2024-11-12 10:53:28,601 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-12 10:53:28,613 main DEBUG Took 0.010089 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 10:53:28,613 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 10:53:28,614 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 10:53:28,615 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 10:53:28,616 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:53:28,623 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 10:53:28,636 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:53:28,638 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:53:28,638 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:53:28,639 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:53:28,639 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:53:28,640 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:53:28,641 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:53:28,641 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:53:28,642 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:53:28,642 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:53:28,643 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:53:28,643 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:53:28,643 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:53:28,644 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:53:28,644 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:53:28,645 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:53:28,645 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:53:28,645 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:53:28,646 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:53:28,646 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:53:28,647 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:53:28,647 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:53:28,648 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:53:28,648 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:53:28,648 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:53:28,649 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 10:53:28,650 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:53:28,651 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 10:53:28,653 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 10:53:28,654 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 10:53:28,655 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 10:53:28,655 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 10:53:28,664 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 10:53:28,666 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 10:53:28,668 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 10:53:28,668 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 10:53:28,669 main DEBUG createAppenders(={Console}) 2024-11-12 10:53:28,669 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-12 10:53:28,670 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-12 10:53:28,670 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-12 10:53:28,671 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 10:53:28,671 main DEBUG OutputStream closed 2024-11-12 10:53:28,671 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 10:53:28,671 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 10:53:28,672 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-12 10:53:28,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 10:53:28,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 10:53:28,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 10:53:28,747 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 10:53:28,748 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 10:53:28,748 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 10:53:28,749 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 10:53:28,749 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 10:53:28,749 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 10:53:28,750 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 10:53:28,750 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 10:53:28,750 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 10:53:28,751 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 10:53:28,751 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 10:53:28,751 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 10:53:28,752 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 10:53:28,752 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 10:53:28,753 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 10:53:28,755 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 10:53:28,755 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-12 10:53:28,756 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 10:53:28,756 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-12T10:53:28,967 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e 2024-11-12 10:53:28,970 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 10:53:28,971 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T10:53:28,979 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-12T10:53:29,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T10:53:29,015 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/cluster_a82bd1d2-3d77-b394-fc32-98907a12be25, deleteOnExit=true 2024-11-12T10:53:29,015 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T10:53:29,016 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/test.cache.data in system properties and HBase conf 2024-11-12T10:53:29,017 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T10:53:29,017 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/hadoop.log.dir in system properties and HBase conf 2024-11-12T10:53:29,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T10:53:29,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T10:53:29,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T10:53:29,098 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T10:53:29,185 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T10:53:29,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T10:53:29,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T10:53:29,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T10:53:29,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T10:53:29,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T10:53:29,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T10:53:29,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T10:53:29,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T10:53:29,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T10:53:29,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/nfs.dump.dir in system properties and HBase conf 2024-11-12T10:53:29,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/java.io.tmpdir in system properties and HBase conf 2024-11-12T10:53:29,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T10:53:29,194 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T10:53:29,194 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T10:53:29,933 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T10:53:30,008 INFO [Time-limited test {}] log.Log(170): Logging initialized @2056ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T10:53:30,076 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T10:53:30,142 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T10:53:30,163 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T10:53:30,163 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T10:53:30,164 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T10:53:30,176 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T10:53:30,178 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/hadoop.log.dir/,AVAILABLE} 2024-11-12T10:53:30,179 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T10:53:30,363 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/java.io.tmpdir/jetty-localhost-39989-hadoop-hdfs-3_4_1-tests_jar-_-any-15333042254634385760/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T10:53:30,369 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:39989} 2024-11-12T10:53:30,369 INFO [Time-limited test {}] server.Server(415): Started @2418ms 2024-11-12T10:53:30,690 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T10:53:30,699 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T10:53:30,700 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T10:53:30,701 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T10:53:30,701 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T10:53:30,702 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/hadoop.log.dir/,AVAILABLE} 2024-11-12T10:53:30,702 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T10:53:30,815 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/java.io.tmpdir/jetty-localhost-34111-hadoop-hdfs-3_4_1-tests_jar-_-any-11643171154355961257/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T10:53:30,816 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:34111} 2024-11-12T10:53:30,816 INFO [Time-limited test {}] server.Server(415): Started @2865ms 2024-11-12T10:53:30,870 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T10:53:31,285 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/cluster_a82bd1d2-3d77-b394-fc32-98907a12be25/data/data1/current/BP-1014935059-172.17.0.2-1731408809727/current, will proceed with Du for space computation calculation, 2024-11-12T10:53:31,285 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/cluster_a82bd1d2-3d77-b394-fc32-98907a12be25/data/data2/current/BP-1014935059-172.17.0.2-1731408809727/current, will proceed with Du for space computation calculation, 2024-11-12T10:53:31,320 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T10:53:31,380 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e 2024-11-12T10:53:31,381 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcd20e6868e636548 with lease ID 0xe3c609c8b79afcd4: Processing first storage report for DS-9dd300ce-c1aa-402d-8817-fe28de508f7c from datanode DatanodeRegistration(127.0.0.1:43441, datanodeUuid=901d1eb8-d5ba-4dfd-bff8-76a6f9bf2888, infoPort=45955, infoSecurePort=0, ipcPort=42889, storageInfo=lv=-57;cid=testClusterID;nsid=1468788542;c=1731408809727) 2024-11-12T10:53:31,382 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcd20e6868e636548 with lease ID 0xe3c609c8b79afcd4: from storage DS-9dd300ce-c1aa-402d-8817-fe28de508f7c node DatanodeRegistration(127.0.0.1:43441, datanodeUuid=901d1eb8-d5ba-4dfd-bff8-76a6f9bf2888, infoPort=45955, infoSecurePort=0, ipcPort=42889, storageInfo=lv=-57;cid=testClusterID;nsid=1468788542;c=1731408809727), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-12T10:53:31,383 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcd20e6868e636548 with lease ID 0xe3c609c8b79afcd4: Processing first storage report for DS-030254d6-6878-49c8-ae63-abcf4ad460ea from datanode DatanodeRegistration(127.0.0.1:43441, datanodeUuid=901d1eb8-d5ba-4dfd-bff8-76a6f9bf2888, infoPort=45955, infoSecurePort=0, ipcPort=42889, storageInfo=lv=-57;cid=testClusterID;nsid=1468788542;c=1731408809727) 2024-11-12T10:53:31,383 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcd20e6868e636548 with lease ID 0xe3c609c8b79afcd4: from storage DS-030254d6-6878-49c8-ae63-abcf4ad460ea node DatanodeRegistration(127.0.0.1:43441, datanodeUuid=901d1eb8-d5ba-4dfd-bff8-76a6f9bf2888, infoPort=45955, infoSecurePort=0, ipcPort=42889, storageInfo=lv=-57;cid=testClusterID;nsid=1468788542;c=1731408809727), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T10:53:31,450 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/cluster_a82bd1d2-3d77-b394-fc32-98907a12be25/zookeeper_0, clientPort=59192, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/cluster_a82bd1d2-3d77-b394-fc32-98907a12be25/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/cluster_a82bd1d2-3d77-b394-fc32-98907a12be25/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T10:53:31,460 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59192 2024-11-12T10:53:31,475 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:53:31,480 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:53:31,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741825_1001 (size=7) 2024-11-12T10:53:32,101 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98 with version=8 2024-11-12T10:53:32,101 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/hbase-staging 2024-11-12T10:53:32,191 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T10:53:32,392 INFO [Time-limited test {}] client.ConnectionUtils(128): master/aed202db7571:0 server-side Connection retries=45 2024-11-12T10:53:32,401 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T10:53:32,402 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T10:53:32,406 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T10:53:32,406 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T10:53:32,406 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T10:53:32,537 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T10:53:32,595 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T10:53:32,603 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T10:53:32,607 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T10:53:32,630 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 43849 (auto-detected) 2024-11-12T10:53:32,631 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-12T10:53:32,649 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45901 2024-11-12T10:53:32,671 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45901 connecting to ZooKeeper ensemble=127.0.0.1:59192 2024-11-12T10:53:32,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:459010x0, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T10:53:32,697 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45901-0x100339132190000 connected 2024-11-12T10:53:32,725 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:53:32,727 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:53:32,737 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T10:53:32,740 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98, hbase.cluster.distributed=false 2024-11-12T10:53:32,762 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T10:53:32,767 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45901 2024-11-12T10:53:32,767 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45901 2024-11-12T10:53:32,768 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45901 2024-11-12T10:53:32,768 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45901 2024-11-12T10:53:32,768 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45901 2024-11-12T10:53:32,868 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/aed202db7571:0 server-side Connection retries=45 2024-11-12T10:53:32,870 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T10:53:32,870 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T10:53:32,870 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T10:53:32,870 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T10:53:32,871 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T10:53:32,873 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T10:53:32,876 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T10:53:32,877 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43143 2024-11-12T10:53:32,879 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43143 connecting to ZooKeeper ensemble=127.0.0.1:59192 2024-11-12T10:53:32,880 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:53:32,883 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:53:32,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:431430x0, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T10:53:32,890 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43143-0x100339132190001 connected 2024-11-12T10:53:32,890 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:431430x0, quorum=127.0.0.1:59192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T10:53:32,895 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T10:53:32,903 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T10:53:32,906 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T10:53:32,912 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T10:53:32,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43143 2024-11-12T10:53:32,913 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43143 2024-11-12T10:53:32,913 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43143 2024-11-12T10:53:32,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43143 2024-11-12T10:53:32,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43143 2024-11-12T10:53:32,931 DEBUG [M:0;aed202db7571:45901 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;aed202db7571:45901 2024-11-12T10:53:32,932 INFO [master/aed202db7571:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/aed202db7571,45901,1731408812240 2024-11-12T10:53:32,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T10:53:32,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T10:53:32,940 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/aed202db7571,45901,1731408812240 2024-11-12T10:53:32,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T10:53:32,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:53:32,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:53:32,962 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T10:53:32,963 INFO [master/aed202db7571:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/aed202db7571,45901,1731408812240 from backup master directory 2024-11-12T10:53:32,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T10:53:32,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/aed202db7571,45901,1731408812240 2024-11-12T10:53:32,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T10:53:32,966 WARN [master/aed202db7571:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T10:53:32,967 INFO [master/aed202db7571:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=aed202db7571,45901,1731408812240 2024-11-12T10:53:32,969 INFO [master/aed202db7571:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T10:53:32,970 INFO [master/aed202db7571:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T10:53:33,024 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/hbase.id] with ID: 8c5c7f0a-4e23-4f9f-b145-59d201be2441 2024-11-12T10:53:33,024 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/.tmp/hbase.id 2024-11-12T10:53:33,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741826_1002 (size=42) 2024-11-12T10:53:33,440 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/.tmp/hbase.id]:[hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/hbase.id] 2024-11-12T10:53:33,492 INFO [master/aed202db7571:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:53:33,497 INFO [master/aed202db7571:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T10:53:33,516 INFO [master/aed202db7571:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-12T10:53:33,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:53:33,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:53:33,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741827_1003 (size=196) 2024-11-12T10:53:33,954 INFO [master/aed202db7571:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T10:53:33,957 INFO [master/aed202db7571:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T10:53:33,975 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:33,980 INFO [master/aed202db7571:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:53:34,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741828_1004 (size=1189) 2024-11-12T10:53:34,434 INFO [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store 2024-11-12T10:53:34,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741829_1005 (size=34) 2024-11-12T10:53:34,863 INFO [master/aed202db7571:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T10:53:34,867 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:53:34,868 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T10:53:34,868 INFO [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:53:34,869 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:53:34,870 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T10:53:34,871 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:53:34,871 INFO [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:53:34,872 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731408814868Disabling compacts and flushes for region at 1731408814868Disabling writes for close at 1731408814870 (+2 ms)Writing region close event to WAL at 1731408814871 (+1 ms)Closed at 1731408814871 2024-11-12T10:53:34,874 WARN [master/aed202db7571:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/.initializing 2024-11-12T10:53:34,875 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/WALs/aed202db7571,45901,1731408812240 2024-11-12T10:53:34,883 INFO [master/aed202db7571:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:53:34,897 INFO [master/aed202db7571:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aed202db7571%2C45901%2C1731408812240, suffix=, logDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/WALs/aed202db7571,45901,1731408812240, archiveDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/oldWALs, maxLogs=10 2024-11-12T10:53:34,918 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/WALs/aed202db7571,45901,1731408812240/aed202db7571%2C45901%2C1731408812240.1731408814901, exclude list is [], retry=0 2024-11-12T10:53:34,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43441,DS-9dd300ce-c1aa-402d-8817-fe28de508f7c,DISK] 2024-11-12T10:53:34,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T10:53:34,971 INFO [master/aed202db7571:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/WALs/aed202db7571,45901,1731408812240/aed202db7571%2C45901%2C1731408812240.1731408814901 2024-11-12T10:53:34,972 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45955:45955)] 2024-11-12T10:53:34,972 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:53:34,973 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:53:34,976 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:53:34,976 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:53:35,010 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:53:35,033 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T10:53:35,036 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:35,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:53:35,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:53:35,042 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T10:53:35,043 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:35,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:53:35,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:53:35,046 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T10:53:35,047 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:35,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:53:35,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:53:35,051 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T10:53:35,051 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:35,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:53:35,052 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:53:35,055 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:53:35,056 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:53:35,062 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:53:35,062 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:53:35,066 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T10:53:35,069 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:53:35,074 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:53:35,075 INFO [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72672454, jitterRate=0.08290395140647888}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T10:53:35,082 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731408814987Initializing all the Stores at 1731408814990 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731408814990Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408814991 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408814991Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408814991Cleaning up temporary data from old regions at 1731408815062 (+71 ms)Region opened successfully at 1731408815081 (+19 ms) 2024-11-12T10:53:35,083 INFO [master/aed202db7571:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T10:53:35,112 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@312b30af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aed202db7571/172.17.0.2:0 2024-11-12T10:53:35,140 INFO [master/aed202db7571:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T10:53:35,150 INFO [master/aed202db7571:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T10:53:35,150 INFO [master/aed202db7571:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T10:53:35,152 INFO [master/aed202db7571:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T10:53:35,154 INFO [master/aed202db7571:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T10:53:35,158 INFO [master/aed202db7571:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-12T10:53:35,158 INFO [master/aed202db7571:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T10:53:35,182 INFO [master/aed202db7571:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T10:53:35,190 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T10:53:35,192 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T10:53:35,194 INFO [master/aed202db7571:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T10:53:35,196 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T10:53:35,198 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T10:53:35,200 INFO [master/aed202db7571:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T10:53:35,203 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T10:53:35,204 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T10:53:35,206 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T10:53:35,207 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T10:53:35,224 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T10:53:35,225 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T10:53:35,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T10:53:35,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T10:53:35,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:53:35,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:53:35,232 INFO [master/aed202db7571:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=aed202db7571,45901,1731408812240, sessionid=0x100339132190000, setting cluster-up flag (Was=false) 2024-11-12T10:53:35,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:53:35,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:53:35,245 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T10:53:35,247 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=aed202db7571,45901,1731408812240 2024-11-12T10:53:35,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:53:35,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:53:35,256 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T10:53:35,258 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=aed202db7571,45901,1731408812240 2024-11-12T10:53:35,265 INFO [master/aed202db7571:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T10:53:35,322 INFO [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(746): ClusterId : 8c5c7f0a-4e23-4f9f-b145-59d201be2441 2024-11-12T10:53:35,324 DEBUG [RS:0;aed202db7571:43143 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T10:53:35,329 DEBUG [RS:0;aed202db7571:43143 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T10:53:35,329 DEBUG [RS:0;aed202db7571:43143 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T10:53:35,331 DEBUG [RS:0;aed202db7571:43143 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T10:53:35,332 DEBUG [RS:0;aed202db7571:43143 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c49f26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aed202db7571/172.17.0.2:0 2024-11-12T10:53:35,332 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T10:53:35,343 INFO [master/aed202db7571:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T10:53:35,347 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;aed202db7571:43143 2024-11-12T10:53:35,350 INFO [RS:0;aed202db7571:43143 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T10:53:35,350 INFO [master/aed202db7571:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T10:53:35,351 INFO [RS:0;aed202db7571:43143 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T10:53:35,351 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T10:53:35,353 INFO [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(2659): reportForDuty to master=aed202db7571,45901,1731408812240 with port=43143, startcode=1731408812834 2024-11-12T10:53:35,357 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: aed202db7571,45901,1731408812240 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T10:53:35,364 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/aed202db7571:0, corePoolSize=5, maxPoolSize=5 2024-11-12T10:53:35,364 DEBUG [RS:0;aed202db7571:43143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T10:53:35,364 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/aed202db7571:0, corePoolSize=5, maxPoolSize=5 2024-11-12T10:53:35,364 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/aed202db7571:0, corePoolSize=5, maxPoolSize=5 2024-11-12T10:53:35,365 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/aed202db7571:0, corePoolSize=5, maxPoolSize=5 2024-11-12T10:53:35,365 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/aed202db7571:0, corePoolSize=10, maxPoolSize=10 2024-11-12T10:53:35,365 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/aed202db7571:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:53:35,365 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/aed202db7571:0, corePoolSize=2, maxPoolSize=2 2024-11-12T10:53:35,365 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/aed202db7571:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:53:35,367 INFO [master/aed202db7571:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731408845367 2024-11-12T10:53:35,369 INFO [master/aed202db7571:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T10:53:35,370 INFO [master/aed202db7571:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T10:53:35,371 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T10:53:35,371 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T10:53:35,374 INFO [master/aed202db7571:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T10:53:35,374 INFO [master/aed202db7571:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T10:53:35,375 INFO [master/aed202db7571:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T10:53:35,375 INFO [master/aed202db7571:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T10:53:35,377 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:35,377 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T10:53:35,376 INFO [master/aed202db7571:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:35,381 INFO [master/aed202db7571:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T10:53:35,382 INFO [master/aed202db7571:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T10:53:35,383 INFO [master/aed202db7571:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T10:53:35,385 INFO [master/aed202db7571:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T10:53:35,385 INFO [master/aed202db7571:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T10:53:35,390 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/aed202db7571:0:becomeActiveMaster-HFileCleaner.large.0-1731408815386,5,FailOnTimeoutGroup] 2024-11-12T10:53:35,391 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/aed202db7571:0:becomeActiveMaster-HFileCleaner.small.0-1731408815390,5,FailOnTimeoutGroup] 2024-11-12T10:53:35,391 INFO [master/aed202db7571:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:35,391 INFO [master/aed202db7571:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T10:53:35,392 INFO [master/aed202db7571:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:35,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741831_1007 (size=1321) 2024-11-12T10:53:35,393 INFO [master/aed202db7571:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:35,395 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T10:53:35,395 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98 2024-11-12T10:53:35,401 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54779, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T10:53:35,407 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45901 {}] master.ServerManager(363): Checking decommissioned status of RegionServer aed202db7571,43143,1731408812834 2024-11-12T10:53:35,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741832_1008 (size=32) 2024-11-12T10:53:35,410 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45901 {}] master.ServerManager(517): Registering regionserver=aed202db7571,43143,1731408812834 2024-11-12T10:53:35,424 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98 2024-11-12T10:53:35,424 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37247 2024-11-12T10:53:35,424 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T10:53:35,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T10:53:35,429 DEBUG [RS:0;aed202db7571:43143 {}] zookeeper.ZKUtil(111): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/aed202db7571,43143,1731408812834 2024-11-12T10:53:35,429 WARN [RS:0;aed202db7571:43143 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T10:53:35,429 INFO [RS:0;aed202db7571:43143 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:53:35,429 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/WALs/aed202db7571,43143,1731408812834 2024-11-12T10:53:35,431 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [aed202db7571,43143,1731408812834] 2024-11-12T10:53:35,454 INFO [RS:0;aed202db7571:43143 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T10:53:35,471 INFO [RS:0;aed202db7571:43143 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T10:53:35,476 INFO [RS:0;aed202db7571:43143 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T10:53:35,476 INFO [RS:0;aed202db7571:43143 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:35,477 INFO [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T10:53:35,482 INFO [RS:0;aed202db7571:43143 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T10:53:35,484 INFO [RS:0;aed202db7571:43143 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:35,484 DEBUG [RS:0;aed202db7571:43143 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/aed202db7571:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:53:35,484 DEBUG [RS:0;aed202db7571:43143 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/aed202db7571:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:53:35,484 DEBUG [RS:0;aed202db7571:43143 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/aed202db7571:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:53:35,484 DEBUG [RS:0;aed202db7571:43143 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/aed202db7571:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:53:35,485 DEBUG [RS:0;aed202db7571:43143 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/aed202db7571:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:53:35,485 DEBUG [RS:0;aed202db7571:43143 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/aed202db7571:0, corePoolSize=2, maxPoolSize=2 2024-11-12T10:53:35,485 DEBUG [RS:0;aed202db7571:43143 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:53:35,485 DEBUG [RS:0;aed202db7571:43143 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/aed202db7571:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:53:35,485 DEBUG [RS:0;aed202db7571:43143 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/aed202db7571:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:53:35,485 DEBUG [RS:0;aed202db7571:43143 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/aed202db7571:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:53:35,485 DEBUG [RS:0;aed202db7571:43143 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/aed202db7571:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:53:35,486 DEBUG [RS:0;aed202db7571:43143 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/aed202db7571:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:53:35,486 DEBUG [RS:0;aed202db7571:43143 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/aed202db7571:0, corePoolSize=3, maxPoolSize=3 2024-11-12T10:53:35,486 DEBUG [RS:0;aed202db7571:43143 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0, corePoolSize=3, maxPoolSize=3 2024-11-12T10:53:35,487 INFO [RS:0;aed202db7571:43143 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:35,487 INFO [RS:0;aed202db7571:43143 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:35,487 INFO [RS:0;aed202db7571:43143 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:35,487 INFO [RS:0;aed202db7571:43143 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:35,487 INFO [RS:0;aed202db7571:43143 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:35,488 INFO [RS:0;aed202db7571:43143 {}] hbase.ChoreService(168): Chore ScheduledChore name=aed202db7571,43143,1731408812834-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T10:53:35,506 INFO [RS:0;aed202db7571:43143 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T10:53:35,507 INFO [RS:0;aed202db7571:43143 {}] hbase.ChoreService(168): Chore ScheduledChore name=aed202db7571,43143,1731408812834-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:35,508 INFO [RS:0;aed202db7571:43143 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:35,508 INFO [RS:0;aed202db7571:43143 {}] regionserver.Replication(171): aed202db7571,43143,1731408812834 started 2024-11-12T10:53:35,525 INFO [RS:0;aed202db7571:43143 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:35,526 INFO [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1482): Serving as aed202db7571,43143,1731408812834, RpcServer on aed202db7571/172.17.0.2:43143, sessionid=0x100339132190001 2024-11-12T10:53:35,527 DEBUG [RS:0;aed202db7571:43143 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T10:53:35,527 DEBUG [RS:0;aed202db7571:43143 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager aed202db7571,43143,1731408812834 2024-11-12T10:53:35,527 DEBUG [RS:0;aed202db7571:43143 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aed202db7571,43143,1731408812834' 2024-11-12T10:53:35,527 DEBUG [RS:0;aed202db7571:43143 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T10:53:35,528 DEBUG [RS:0;aed202db7571:43143 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T10:53:35,529 DEBUG [RS:0;aed202db7571:43143 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T10:53:35,529 DEBUG [RS:0;aed202db7571:43143 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T10:53:35,529 DEBUG [RS:0;aed202db7571:43143 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager aed202db7571,43143,1731408812834 2024-11-12T10:53:35,529 DEBUG [RS:0;aed202db7571:43143 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aed202db7571,43143,1731408812834' 2024-11-12T10:53:35,529 DEBUG [RS:0;aed202db7571:43143 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T10:53:35,530 DEBUG [RS:0;aed202db7571:43143 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T10:53:35,530 DEBUG [RS:0;aed202db7571:43143 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T10:53:35,530 INFO [RS:0;aed202db7571:43143 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T10:53:35,531 INFO [RS:0;aed202db7571:43143 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T10:53:35,645 INFO [RS:0;aed202db7571:43143 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:53:35,648 INFO [RS:0;aed202db7571:43143 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aed202db7571%2C43143%2C1731408812834, suffix=, logDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/WALs/aed202db7571,43143,1731408812834, archiveDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/oldWALs, maxLogs=32 2024-11-12T10:53:35,664 DEBUG [RS:0;aed202db7571:43143 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/WALs/aed202db7571,43143,1731408812834/aed202db7571%2C43143%2C1731408812834.1731408815650, exclude list is [], retry=0 2024-11-12T10:53:35,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43441,DS-9dd300ce-c1aa-402d-8817-fe28de508f7c,DISK] 2024-11-12T10:53:35,671 INFO [RS:0;aed202db7571:43143 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/WALs/aed202db7571,43143,1731408812834/aed202db7571%2C43143%2C1731408812834.1731408815650 2024-11-12T10:53:35,672 DEBUG [RS:0;aed202db7571:43143 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45955:45955)] 2024-11-12T10:53:35,814 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:53:35,820 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T10:53:35,826 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T10:53:35,827 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:35,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:53:35,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T10:53:35,832 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T10:53:35,832 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:35,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:53:35,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T10:53:35,836 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T10:53:35,836 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:35,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:53:35,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T10:53:35,840 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T10:53:35,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:35,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:53:35,842 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T10:53:35,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740 2024-11-12T10:53:35,844 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740 2024-11-12T10:53:35,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T10:53:35,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T10:53:35,849 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T10:53:35,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T10:53:35,856 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:53:35,857 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59260061, jitterRate=-0.11695627868175507}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T10:53:35,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731408815815Initializing all the Stores at 1731408815819 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731408815819Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731408815820 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408815820Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731408815820Cleaning up temporary data from old regions at 1731408815848 (+28 ms)Region opened successfully at 1731408815860 (+12 ms) 2024-11-12T10:53:35,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T10:53:35,860 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T10:53:35,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T10:53:35,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T10:53:35,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T10:53:35,862 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T10:53:35,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731408815860Disabling compacts and flushes for region at 1731408815860Disabling writes for close at 1731408815860Writing region close event to WAL at 1731408815861 (+1 ms)Closed at 1731408815862 (+1 ms) 2024-11-12T10:53:35,866 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T10:53:35,866 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T10:53:35,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T10:53:35,880 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T10:53:35,883 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T10:53:36,037 DEBUG [aed202db7571:45901 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T10:53:36,051 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:53:36,059 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as aed202db7571,43143,1731408812834, state=OPENING 2024-11-12T10:53:36,063 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T10:53:36,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:53:36,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:53:36,066 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:53:36,066 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:53:36,067 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T10:53:36,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:53:36,241 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T10:53:36,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47719, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T10:53:36,256 INFO [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T10:53:36,256 INFO [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:53:36,257 INFO [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T10:53:36,261 INFO [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aed202db7571%2C43143%2C1731408812834.meta, suffix=.meta, logDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/WALs/aed202db7571,43143,1731408812834, archiveDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/oldWALs, maxLogs=32 2024-11-12T10:53:36,275 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/WALs/aed202db7571,43143,1731408812834/aed202db7571%2C43143%2C1731408812834.meta.1731408816262.meta, exclude list is [], retry=0 2024-11-12T10:53:36,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43441,DS-9dd300ce-c1aa-402d-8817-fe28de508f7c,DISK] 2024-11-12T10:53:36,282 INFO [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/WALs/aed202db7571,43143,1731408812834/aed202db7571%2C43143%2C1731408812834.meta.1731408816262.meta 2024-11-12T10:53:36,282 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45955:45955)] 2024-11-12T10:53:36,282 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:53:36,284 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T10:53:36,286 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T10:53:36,291 INFO [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T10:53:36,294 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T10:53:36,295 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:53:36,295 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T10:53:36,295 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T10:53:36,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T10:53:36,300 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T10:53:36,300 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:36,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:53:36,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T10:53:36,302 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T10:53:36,302 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:36,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:53:36,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T10:53:36,305 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T10:53:36,305 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:36,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:53:36,306 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T10:53:36,307 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T10:53:36,307 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:36,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:53:36,308 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T10:53:36,309 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740 2024-11-12T10:53:36,312 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740 2024-11-12T10:53:36,315 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T10:53:36,315 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T10:53:36,316 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T10:53:36,318 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T10:53:36,320 INFO [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71761175, jitterRate=0.06932483613491058}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T10:53:36,320 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T10:53:36,322 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731408816296Writing region info on filesystem at 1731408816296Initializing all the Stores at 1731408816298 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731408816298Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731408816298Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408816298Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731408816298Cleaning up temporary data from old regions at 1731408816315 (+17 ms)Running coprocessor post-open hooks at 1731408816321 (+6 ms)Region opened successfully at 1731408816322 (+1 ms) 2024-11-12T10:53:36,328 INFO [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731408816234 2024-11-12T10:53:36,338 DEBUG [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T10:53:36,339 INFO [RS_OPEN_META-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T10:53:36,340 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:53:36,342 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as aed202db7571,43143,1731408812834, state=OPEN 2024-11-12T10:53:36,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T10:53:36,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T10:53:36,346 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:53:36,346 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:53:36,346 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:36,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T10:53:36,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=aed202db7571,43143,1731408812834 in 277 msec 2024-11-12T10:53:36,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T10:53:36,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 482 msec 2024-11-12T10:53:36,363 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T10:53:36,363 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T10:53:36,384 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:53:36,386 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:53:36,407 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:53:36,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38715, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:53:36,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1330 sec 2024-11-12T10:53:36,429 INFO [master/aed202db7571:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731408816429, completionTime=-1 2024-11-12T10:53:36,431 INFO [master/aed202db7571:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-12T10:53:36,432 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T10:53:36,455 INFO [master/aed202db7571:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-12T10:53:36,456 INFO [master/aed202db7571:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731408876456 2024-11-12T10:53:36,456 INFO [master/aed202db7571:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731408936456 2024-11-12T10:53:36,456 INFO [master/aed202db7571:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-12T10:53:36,458 INFO [master/aed202db7571:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aed202db7571,45901,1731408812240-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:36,459 INFO [master/aed202db7571:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aed202db7571,45901,1731408812240-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:36,459 INFO [master/aed202db7571:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aed202db7571,45901,1731408812240-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:36,460 INFO [master/aed202db7571:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-aed202db7571:45901, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:36,461 INFO [master/aed202db7571:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:36,461 INFO [master/aed202db7571:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:36,467 DEBUG [master/aed202db7571:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T10:53:36,490 INFO [master/aed202db7571:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.522sec 2024-11-12T10:53:36,491 INFO [master/aed202db7571:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T10:53:36,493 INFO [master/aed202db7571:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T10:53:36,494 INFO [master/aed202db7571:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T10:53:36,494 INFO [master/aed202db7571:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T10:53:36,494 INFO [master/aed202db7571:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T10:53:36,495 INFO [master/aed202db7571:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aed202db7571,45901,1731408812240-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T10:53:36,496 INFO [master/aed202db7571:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aed202db7571,45901,1731408812240-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T10:53:36,504 DEBUG [master/aed202db7571:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T10:53:36,505 INFO [master/aed202db7571:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T10:53:36,506 INFO [master/aed202db7571:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aed202db7571,45901,1731408812240-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:53:36,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:36,538 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T10:53:36,538 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T10:53:36,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:53:36,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:53:36,550 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:53:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:53:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:53:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775efbc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:53:36,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:53:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:36,560 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:53:36,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:36,564 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:53:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:53:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:53:36,574 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:53:36,593 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=aed202db7571,45901,1731408812240 2024-11-12T10:53:36,605 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=737, ProcessCount=11, AvailableMemoryMB=2779 2024-11-12T10:53:36,613 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T10:53:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is aed202db7571,45901,1731408812240 2024-11-12T10:53:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c35b008 2024-11-12T10:53:36,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T10:53:36,646 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T10:53:36,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T10:53:36,656 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T10:53:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T10:53:36,674 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T10:53:36,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-12T10:53:36,677 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:36,680 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T10:53:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T10:53:36,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741835_1011 (size=1061) 2024-11-12T10:53:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T10:53:37,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T10:53:37,099 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98 2024-11-12T10:53:37,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741836_1012 (size=53) 2024-11-12T10:53:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T10:53:37,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:53:37,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b3cb1a6227d163d89478d6e9738e8055, disabling compactions & flushes 2024-11-12T10:53:37,511 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:37,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:37,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. after waiting 0 ms 2024-11-12T10:53:37,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:37,511 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:37,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b3cb1a6227d163d89478d6e9738e8055: Waiting for close lock at 1731408817511Disabling compacts and flushes for region at 1731408817511Disabling writes for close at 1731408817511Writing region close event to WAL at 1731408817511Closed at 1731408817511 2024-11-12T10:53:37,513 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T10:53:37,518 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731408817514"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731408817514"}]},"ts":"1731408817514"} 2024-11-12T10:53:37,525 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T10:53:37,527 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T10:53:37,530 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408817528"}]},"ts":"1731408817528"} 2024-11-12T10:53:37,535 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T10:53:37,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cb1a6227d163d89478d6e9738e8055, ASSIGN}] 2024-11-12T10:53:37,539 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cb1a6227d163d89478d6e9738e8055, ASSIGN 2024-11-12T10:53:37,541 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cb1a6227d163d89478d6e9738e8055, ASSIGN; state=OFFLINE, location=aed202db7571,43143,1731408812834; forceNewPlan=false, retain=false 2024-11-12T10:53:37,694 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b3cb1a6227d163d89478d6e9738e8055, regionState=OPENING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:53:37,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cb1a6227d163d89478d6e9738e8055, ASSIGN because future has completed 2024-11-12T10:53:37,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:53:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T10:53:37,867 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:37,868 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:53:37,868 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:37,868 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:53:37,869 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:37,869 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:37,873 INFO [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:37,877 INFO [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:53:37,878 INFO [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3cb1a6227d163d89478d6e9738e8055 columnFamilyName A 2024-11-12T10:53:37,878 DEBUG [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:37,879 INFO [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] regionserver.HStore(327): Store=b3cb1a6227d163d89478d6e9738e8055/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:53:37,879 INFO [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:37,882 INFO [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:53:37,882 INFO [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3cb1a6227d163d89478d6e9738e8055 columnFamilyName B 2024-11-12T10:53:37,883 DEBUG [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:37,884 INFO [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] regionserver.HStore(327): Store=b3cb1a6227d163d89478d6e9738e8055/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:53:37,884 INFO [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:37,886 INFO [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:53:37,887 INFO [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3cb1a6227d163d89478d6e9738e8055 columnFamilyName C 2024-11-12T10:53:37,887 DEBUG [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:53:37,888 INFO [StoreOpener-b3cb1a6227d163d89478d6e9738e8055-1 {}] regionserver.HStore(327): Store=b3cb1a6227d163d89478d6e9738e8055/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:53:37,888 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:37,888 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:37,890 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:37,890 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:37,892 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:37,892 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:37,893 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T10:53:37,896 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:37,899 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:53:37,900 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b3cb1a6227d163d89478d6e9738e8055; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61377142, jitterRate=-0.08540931344032288}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T10:53:37,900 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:37,901 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b3cb1a6227d163d89478d6e9738e8055: Running coprocessor pre-open hook at 1731408817869Writing region info on filesystem at 1731408817869Initializing all the Stores at 1731408817872 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408817872Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408817872Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408817872Cleaning up temporary data from old regions at 1731408817892 (+20 ms)Running coprocessor post-open hooks at 1731408817900 (+8 ms)Region opened successfully at 1731408817901 (+1 ms) 2024-11-12T10:53:37,903 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., pid=6, masterSystemTime=1731408817858 2024-11-12T10:53:37,906 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:37,906 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:37,908 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b3cb1a6227d163d89478d6e9738e8055, regionState=OPEN, openSeqNum=2, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:53:37,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:53:37,913 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45901 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=aed202db7571,43143,1731408812834, table=TestAcidGuarantees, region=b3cb1a6227d163d89478d6e9738e8055. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T10:53:37,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T10:53:37,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 in 210 msec 2024-11-12T10:53:37,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T10:53:37,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cb1a6227d163d89478d6e9738e8055, ASSIGN in 382 msec 2024-11-12T10:53:37,924 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T10:53:37,924 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408817924"}]},"ts":"1731408817924"} 2024-11-12T10:53:37,927 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T10:53:37,929 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T10:53:37,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2690 sec 2024-11-12T10:53:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T10:53:38,850 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:53:38,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,861 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:53:38,861 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:53:38,863 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:53:38,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:53:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:53:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a04f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:53:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:53:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:38,867 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:53:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435bed7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,869 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:53:38,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:53:38,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:53:38,874 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:53:38,877 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T10:53:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is aed202db7571,45901,1731408812240 2024-11-12T10:53:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6bf45604 2024-11-12T10:53:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T10:53:38,884 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T10:53:38,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:53:38,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:53:38,891 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:53:38,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:53:38,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:53:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3d2fd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:53:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:53:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:38,895 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:53:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c860641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:53:38,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:53:38,899 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:53:38,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:53:38,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:53:38,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787f40de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:53:38,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:53:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:38,903 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:53:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740f65e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:53:38,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:53:38,908 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:53:38,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:53:38,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:53:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b21050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:53:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:53:38,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:38,911 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:53:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79382fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:53:38,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:53:38,914 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:53:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:53:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:53:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45101745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:53:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:53:38,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:38,917 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:53:38,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0d5e57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:53:38,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:53:38,920 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:53:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:53:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:53:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4331d0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:53:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:53:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:38,923 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:53:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17daddbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:53:38,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:53:38,927 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:53:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:53:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:53:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c3ac0b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:53:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:53:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:38,929 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:53:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c90417c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:53:38,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:53:38,932 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:53:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:53:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:53:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9550bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:53:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:53:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:38,935 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:53:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b38edab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:53:38,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:53:38,938 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:53:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:53:38,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:53:38,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc64f57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:53:38,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:53:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:38,941 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:53:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3adce01e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,942 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:53:38,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:53:38,943 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:53:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:53:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:53:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a0271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:53:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:53:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:38,946 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:53:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9402b31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:53:38,952 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:53:38,952 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:53:38,952 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:53:38,952 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:53:38,954 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:53:38,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:53:38,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:53:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:53:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:53:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:53:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:53:38,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:53:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:53:38,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:53:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:53:38,952 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:53:38,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:53:38,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:53:38,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:53:38,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:53:38,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:53:38,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:53:38,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:53:38,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:53:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:53:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:53:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:53:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:53:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:53:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:53:38,972 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:53:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:53:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:53:38,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:53:38,977 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:53:38,979 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:53:38,980 DEBUG [Thread-156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:53:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:53:38,980 DEBUG [Thread-156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:53:38,982 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:53:38,982 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:53:38,983 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:53:38,983 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:53:38,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:53:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:53:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:53:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-12T10:53:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T10:53:38,997 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:53:39,003 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:53:39,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:53:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:39,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T10:53:39,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:39,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:39,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:39,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:39,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:39,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T10:53:39,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/c46d8b0051db40e8998fc7f7f0197c00 is 50, key is test_row_0/A:col10/1731408819030/Put/seqid=0 2024-11-12T10:53:39,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741837_1013 (size=12001) 2024-11-12T10:53:39,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T10:53:39,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:39,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:39,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:39,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731408819188, max timestamp=1731408819263Num uniques 859; 2024-11-12T10:53:39,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T10:53:39,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731408819188, max timestamp=1731408819267Num uniques 859; 2024-11-12T10:53:39,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T10:53:39,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731408819188, max timestamp=1731408819263Num uniques 859; 2024-11-12T10:53:39,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T10:53:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408879239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T10:53:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T10:53:39,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:39,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408879333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408879396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408879396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408879396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408879448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408879508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T10:53:39,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:39,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:39,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:39,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408879519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408879520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408879520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/c46d8b0051db40e8998fc7f7f0197c00 2024-11-12T10:53:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T10:53:39,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/c90362070ef1422786371c514a630206 is 50, key is test_row_0/B:col10/1731408819030/Put/seqid=0 2024-11-12T10:53:39,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408879658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T10:53:39,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:39,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:39,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:39,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741838_1014 (size=12001) 2024-11-12T10:53:39,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408879717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408879738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408879739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408879739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T10:53:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:39,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408879968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:39,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T10:53:39,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:39,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:39,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:39,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:40,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408880028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:40,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:40,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408880047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:40,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408880048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:40,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:40,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408880057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:40,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/c90362070ef1422786371c514a630206 2024-11-12T10:53:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T10:53:40,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/ef43ae91ec7448bf921a525fe97157f8 is 50, key is test_row_0/C:col10/1731408819030/Put/seqid=0 2024-11-12T10:53:40,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T10:53:40,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:40,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:40,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:40,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:40,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:40,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741839_1015 (size=12001) 2024-11-12T10:53:40,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/ef43ae91ec7448bf921a525fe97157f8 2024-11-12T10:53:40,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/c46d8b0051db40e8998fc7f7f0197c00 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/c46d8b0051db40e8998fc7f7f0197c00 2024-11-12T10:53:40,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/c46d8b0051db40e8998fc7f7f0197c00, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T10:53:40,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/c90362070ef1422786371c514a630206 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c90362070ef1422786371c514a630206 2024-11-12T10:53:40,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c90362070ef1422786371c514a630206, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T10:53:40,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/ef43ae91ec7448bf921a525fe97157f8 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/ef43ae91ec7448bf921a525fe97157f8 2024-11-12T10:53:40,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/ef43ae91ec7448bf921a525fe97157f8, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T10:53:40,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for b3cb1a6227d163d89478d6e9738e8055 in 1203ms, sequenceid=12, compaction requested=false 2024-11-12T10:53:40,256 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T10:53:40,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:40,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T10:53:40,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:40,303 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-12T10:53:40,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:40,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:40,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:40,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:40,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:40,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:40,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/ee80f3ebab8449b9b7b5082dfdbe9855 is 50, key is test_row_0/A:col10/1731408819328/Put/seqid=0 2024-11-12T10:53:40,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741840_1016 (size=12001) 2024-11-12T10:53:40,347 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/ee80f3ebab8449b9b7b5082dfdbe9855 2024-11-12T10:53:40,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/a6112230b1154887867f5ba680f9159e is 50, key is test_row_0/B:col10/1731408819328/Put/seqid=0 2024-11-12T10:53:40,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741841_1017 (size=12001) 2024-11-12T10:53:40,402 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/a6112230b1154887867f5ba680f9159e 2024-11-12T10:53:40,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/26ed2a36f5a14da7a915146c84498228 is 50, key is test_row_0/C:col10/1731408819328/Put/seqid=0 2024-11-12T10:53:40,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741842_1018 (size=12001) 2024-11-12T10:53:40,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:40,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:40,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408880568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:40,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:40,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:40,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408880568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:40,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408880578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:40,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:40,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:40,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408880592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408880590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408880719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:40,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408880723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:40,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:40,857 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/26ed2a36f5a14da7a915146c84498228 2024-11-12T10:53:40,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/ee80f3ebab8449b9b7b5082dfdbe9855 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ee80f3ebab8449b9b7b5082dfdbe9855 2024-11-12T10:53:40,890 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ee80f3ebab8449b9b7b5082dfdbe9855, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T10:53:40,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/a6112230b1154887867f5ba680f9159e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/a6112230b1154887867f5ba680f9159e 2024-11-12T10:53:40,915 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/a6112230b1154887867f5ba680f9159e, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T10:53:40,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/26ed2a36f5a14da7a915146c84498228 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/26ed2a36f5a14da7a915146c84498228 2024-11-12T10:53:40,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408880928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:40,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:40,933 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/26ed2a36f5a14da7a915146c84498228, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T10:53:40,936 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for b3cb1a6227d163d89478d6e9738e8055 in 632ms, sequenceid=48, compaction requested=false 2024-11-12T10:53:40,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:40,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:40,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-12T10:53:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-12T10:53:40,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T10:53:40,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9340 sec 2024-11-12T10:53:40,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T10:53:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:40,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:40,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:40,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:40,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:40,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:40,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:40,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9700 sec 2024-11-12T10:53:40,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/e6e4b24294e640c784d637c912e23f5f is 50, key is test_row_0/A:col10/1731408820945/Put/seqid=0 2024-11-12T10:53:40,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741843_1019 (size=14341) 2024-11-12T10:53:41,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408821058, max timestamp=1731408821115Num uniques 859; 2024-11-12T10:53:41,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:53:41,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408821058, max timestamp=1731408821115Num uniques 859; 2024-11-12T10:53:41,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:53:41,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408821058, max timestamp=1731408821115Num uniques 859; 2024-11-12T10:53:41,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:53:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T10:53:41,139 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:53:41,140 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:53:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-12T10:53:41,144 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:53:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T10:53:41,147 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:53:41,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:53:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T10:53:41,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408881253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:41,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408881257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:41,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:41,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T10:53:41,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:41,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:41,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:41,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:41,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:41,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408881367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:41,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408881368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:41,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:41,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/e6e4b24294e640c784d637c912e23f5f 2024-11-12T10:53:41,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/d4a6dafe7a1847db96a43b8ecf192b4c is 50, key is test_row_0/B:col10/1731408820945/Put/seqid=0 2024-11-12T10:53:41,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741844_1020 (size=12001) 2024-11-12T10:53:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T10:53:41,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T10:53:41,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:41,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:41,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408881577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:41,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408881577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:41,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408881597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:41,605 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in b3cb1a6227d163d89478d6e9738e8055 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:53:41,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:41,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408881608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:41,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408881608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:41,613 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in b3cb1a6227d163d89478d6e9738e8055 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:53:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:41,615 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in b3cb1a6227d163d89478d6e9738e8055 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:53:41,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:41,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T10:53:41,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:41,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:41,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:41,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:41,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T10:53:41,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T10:53:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:41,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:41,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:41,811 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T10:53:41,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/d4a6dafe7a1847db96a43b8ecf192b4c 2024-11-12T10:53:41,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/d360388004a44fd9a38bdb88e31d528c is 50, key is test_row_0/C:col10/1731408820945/Put/seqid=0 2024-11-12T10:53:41,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408881888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:41,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408881888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:41,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741845_1021 (size=12001) 2024-11-12T10:53:41,911 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T10:53:41,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T10:53:41,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:41,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:41,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:41,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:41,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:42,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T10:53:42,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:42,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:42,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:42,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:42,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:42,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T10:53:42,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:42,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:42,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:42,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:42,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:42,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T10:53:42,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/d360388004a44fd9a38bdb88e31d528c 2024-11-12T10:53:42,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/e6e4b24294e640c784d637c912e23f5f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/e6e4b24294e640c784d637c912e23f5f 2024-11-12T10:53:42,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/e6e4b24294e640c784d637c912e23f5f, entries=200, sequenceid=59, filesize=14.0 K 2024-11-12T10:53:42,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/d4a6dafe7a1847db96a43b8ecf192b4c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/d4a6dafe7a1847db96a43b8ecf192b4c 2024-11-12T10:53:42,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/d4a6dafe7a1847db96a43b8ecf192b4c, entries=150, sequenceid=59, filesize=11.7 K 2024-11-12T10:53:42,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/d360388004a44fd9a38bdb88e31d528c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/d360388004a44fd9a38bdb88e31d528c 2024-11-12T10:53:42,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/d360388004a44fd9a38bdb88e31d528c, entries=150, sequenceid=59, filesize=11.7 K 2024-11-12T10:53:42,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b3cb1a6227d163d89478d6e9738e8055 in 1420ms, sequenceid=59, compaction requested=true 2024-11-12T10:53:42,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:42,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:53:42,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:42,376 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:42,376 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:42,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:53:42,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:42,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:53:42,380 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:42,382 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/B is initiating minor compaction (all files) 2024-11-12T10:53:42,383 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/B in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:42,384 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c90362070ef1422786371c514a630206, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/a6112230b1154887867f5ba680f9159e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/d4a6dafe7a1847db96a43b8ecf192b4c] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=35.2 K 2024-11-12T10:53:42,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:42,391 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting c90362070ef1422786371c514a630206, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731408819028 2024-11-12T10:53:42,394 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting a6112230b1154887867f5ba680f9159e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731408819328 2024-11-12T10:53:42,395 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:42,395 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting d4a6dafe7a1847db96a43b8ecf192b4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731408820553 2024-11-12T10:53:42,395 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/A is initiating minor compaction (all files) 2024-11-12T10:53:42,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T10:53:42,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:42,397 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/A in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:42,397 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-12T10:53:42,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:42,398 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/c46d8b0051db40e8998fc7f7f0197c00, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ee80f3ebab8449b9b7b5082dfdbe9855, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/e6e4b24294e640c784d637c912e23f5f] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=37.4 K 2024-11-12T10:53:42,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:42,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:42,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:42,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:42,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:42,399 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting c46d8b0051db40e8998fc7f7f0197c00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731408819028 2024-11-12T10:53:42,400 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee80f3ebab8449b9b7b5082dfdbe9855, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731408819328 2024-11-12T10:53:42,401 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6e4b24294e640c784d637c912e23f5f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731408820553 2024-11-12T10:53:42,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/d92ce7472b8c41639f87132beeb870c5 is 50, key is test_row_0/A:col10/1731408821252/Put/seqid=0 2024-11-12T10:53:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:42,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:42,451 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#B#compaction#11 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:42,452 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/73ca166779584788ae5f2f23c72bfda9 is 50, key is test_row_0/B:col10/1731408820945/Put/seqid=0 2024-11-12T10:53:42,457 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#A#compaction#10 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:42,458 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/74447cab94084619b9f42b3674cb1480 is 50, key is test_row_0/A:col10/1731408820945/Put/seqid=0 2024-11-12T10:53:42,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741846_1022 (size=12001) 2024-11-12T10:53:42,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741848_1024 (size=12104) 2024-11-12T10:53:42,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408882471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:42,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:42,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:42,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:42,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741847_1023 (size=12104) 2024-11-12T10:53:42,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408882483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:42,510 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/73ca166779584788ae5f2f23c72bfda9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/73ca166779584788ae5f2f23c72bfda9 2024-11-12T10:53:42,537 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/B of b3cb1a6227d163d89478d6e9738e8055 into 73ca166779584788ae5f2f23c72bfda9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:42,538 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:42,540 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/B, priority=13, startTime=1731408822376; duration=0sec 2024-11-12T10:53:42,541 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:42,541 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:B 2024-11-12T10:53:42,541 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:42,544 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:42,544 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/C is initiating minor compaction (all files) 2024-11-12T10:53:42,544 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/C in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:42,544 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/ef43ae91ec7448bf921a525fe97157f8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/26ed2a36f5a14da7a915146c84498228, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/d360388004a44fd9a38bdb88e31d528c] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=35.2 K 2024-11-12T10:53:42,548 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ef43ae91ec7448bf921a525fe97157f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731408819028 2024-11-12T10:53:42,550 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 26ed2a36f5a14da7a915146c84498228, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731408819328 2024-11-12T10:53:42,552 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting d360388004a44fd9a38bdb88e31d528c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731408820553 2024-11-12T10:53:42,586 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#C#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:42,588 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/2e924ed5445f4fd999e1853ffcd672fc is 50, key is test_row_0/C:col10/1731408820945/Put/seqid=0 2024-11-12T10:53:42,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T10:53:42,592 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T10:53:42,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408882588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:42,594 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T10:53:42,594 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T10:53:42,595 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T10:53:42,595 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T10:53:42,596 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T10:53:42,596 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-12T10:53:42,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408882598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:42,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741849_1025 (size=12104) 2024-11-12T10:53:42,632 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/2e924ed5445f4fd999e1853ffcd672fc as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/2e924ed5445f4fd999e1853ffcd672fc 2024-11-12T10:53:42,645 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/C of b3cb1a6227d163d89478d6e9738e8055 into 2e924ed5445f4fd999e1853ffcd672fc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:42,645 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:42,645 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/C, priority=13, startTime=1731408822378; duration=0sec 2024-11-12T10:53:42,645 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:42,646 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:C 2024-11-12T10:53:42,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408882798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:42,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408882807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:42,872 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/d92ce7472b8c41639f87132beeb870c5 2024-11-12T10:53:42,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/f3f9424f508548709d7ea185967be478 is 50, key is test_row_0/B:col10/1731408821252/Put/seqid=0 2024-11-12T10:53:42,895 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/74447cab94084619b9f42b3674cb1480 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/74447cab94084619b9f42b3674cb1480 2024-11-12T10:53:42,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741850_1026 (size=12001) 2024-11-12T10:53:42,908 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/f3f9424f508548709d7ea185967be478 2024-11-12T10:53:42,908 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/A of b3cb1a6227d163d89478d6e9738e8055 into 74447cab94084619b9f42b3674cb1480(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:42,908 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:42,908 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/A, priority=13, startTime=1731408822372; duration=0sec 2024-11-12T10:53:42,909 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:42,909 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:A 2024-11-12T10:53:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/bb8e6d29b4b743d79a50763dcacafc7c is 50, key is test_row_0/C:col10/1731408821252/Put/seqid=0 2024-11-12T10:53:42,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741851_1027 (size=12001) 2024-11-12T10:53:43,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408883117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:43,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:43,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408883117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T10:53:43,349 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/bb8e6d29b4b743d79a50763dcacafc7c 2024-11-12T10:53:43,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/d92ce7472b8c41639f87132beeb870c5 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/d92ce7472b8c41639f87132beeb870c5 2024-11-12T10:53:43,378 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/d92ce7472b8c41639f87132beeb870c5, entries=150, sequenceid=95, filesize=11.7 K 2024-11-12T10:53:43,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/f3f9424f508548709d7ea185967be478 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f3f9424f508548709d7ea185967be478 2024-11-12T10:53:43,398 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f3f9424f508548709d7ea185967be478, entries=150, sequenceid=95, filesize=11.7 K 2024-11-12T10:53:43,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/bb8e6d29b4b743d79a50763dcacafc7c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/bb8e6d29b4b743d79a50763dcacafc7c 2024-11-12T10:53:43,416 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/bb8e6d29b4b743d79a50763dcacafc7c, entries=150, sequenceid=95, filesize=11.7 K 2024-11-12T10:53:43,422 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for b3cb1a6227d163d89478d6e9738e8055 in 1024ms, sequenceid=95, compaction requested=false 2024-11-12T10:53:43,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:43,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:43,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-12T10:53:43,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-12T10:53:43,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-12T10:53:43,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2780 sec 2024-11-12T10:53:43,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.2950 sec 2024-11-12T10:53:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:43,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T10:53:43,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:43,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:43,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:43,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:43,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:43,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:43,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/ae41ff35e7194a42ad0ff9ca760e5423 is 50, key is test_row_0/A:col10/1731408823617/Put/seqid=0 2024-11-12T10:53:43,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741852_1028 (size=14341) 2024-11-12T10:53:43,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/ae41ff35e7194a42ad0ff9ca760e5423 2024-11-12T10:53:43,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408883664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:43,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:43,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408883670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408883670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408883674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:43,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408883677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:43,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/8ed1a1443b03415b9c46b8012d19ae8b is 50, key is test_row_0/B:col10/1731408823617/Put/seqid=0 2024-11-12T10:53:43,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741853_1029 (size=12001) 2024-11-12T10:53:43,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408883787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408883788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:43,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408883788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408883788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408883788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408883997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408883998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408883998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408883998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408883998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/8ed1a1443b03415b9c46b8012d19ae8b 2024-11-12T10:53:44,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/0f54bcbb4ff8408d97dea2cbb15bbd33 is 50, key is test_row_0/C:col10/1731408823617/Put/seqid=0 2024-11-12T10:53:44,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741854_1030 (size=12001) 2024-11-12T10:53:44,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408884307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408884307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408884307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408884307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408884317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/0f54bcbb4ff8408d97dea2cbb15bbd33 2024-11-12T10:53:44,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/ae41ff35e7194a42ad0ff9ca760e5423 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ae41ff35e7194a42ad0ff9ca760e5423 2024-11-12T10:53:44,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ae41ff35e7194a42ad0ff9ca760e5423, entries=200, sequenceid=110, filesize=14.0 K 2024-11-12T10:53:44,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/8ed1a1443b03415b9c46b8012d19ae8b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/8ed1a1443b03415b9c46b8012d19ae8b 2024-11-12T10:53:44,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/8ed1a1443b03415b9c46b8012d19ae8b, entries=150, sequenceid=110, filesize=11.7 K 2024-11-12T10:53:44,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/0f54bcbb4ff8408d97dea2cbb15bbd33 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/0f54bcbb4ff8408d97dea2cbb15bbd33 2024-11-12T10:53:44,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/0f54bcbb4ff8408d97dea2cbb15bbd33, entries=150, sequenceid=110, filesize=11.7 K 2024-11-12T10:53:44,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b3cb1a6227d163d89478d6e9738e8055 in 949ms, sequenceid=110, compaction requested=true 2024-11-12T10:53:44,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:44,569 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:44,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:53:44,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:44,570 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:44,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:53:44,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:44,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:53:44,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:44,571 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:44,571 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/A is initiating minor compaction (all files) 2024-11-12T10:53:44,571 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/A in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:44,572 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/74447cab94084619b9f42b3674cb1480, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/d92ce7472b8c41639f87132beeb870c5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ae41ff35e7194a42ad0ff9ca760e5423] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=37.5 K 2024-11-12T10:53:44,572 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:44,572 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/B is initiating minor compaction (all files) 2024-11-12T10:53:44,572 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/B in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:44,572 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/73ca166779584788ae5f2f23c72bfda9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f3f9424f508548709d7ea185967be478, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/8ed1a1443b03415b9c46b8012d19ae8b] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=35.3 K 2024-11-12T10:53:44,573 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74447cab94084619b9f42b3674cb1480, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731408820553 2024-11-12T10:53:44,573 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 73ca166779584788ae5f2f23c72bfda9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731408820553 2024-11-12T10:53:44,573 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting d92ce7472b8c41639f87132beeb870c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731408821213 2024-11-12T10:53:44,574 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting f3f9424f508548709d7ea185967be478, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731408821213 2024-11-12T10:53:44,575 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae41ff35e7194a42ad0ff9ca760e5423, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731408822447 2024-11-12T10:53:44,575 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ed1a1443b03415b9c46b8012d19ae8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731408822447 2024-11-12T10:53:44,592 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:44,593 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/91cadca5c25d47339f787f2ebe2acd99 is 50, key is test_row_0/A:col10/1731408823617/Put/seqid=0 2024-11-12T10:53:44,593 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:44,593 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/61c9d781ae774bc3bccf98f736498698 is 50, key is test_row_0/B:col10/1731408823617/Put/seqid=0 2024-11-12T10:53:44,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741855_1031 (size=12207) 2024-11-12T10:53:44,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741856_1032 (size=12207) 2024-11-12T10:53:44,647 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/61c9d781ae774bc3bccf98f736498698 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/61c9d781ae774bc3bccf98f736498698 2024-11-12T10:53:44,660 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/B of b3cb1a6227d163d89478d6e9738e8055 into 61c9d781ae774bc3bccf98f736498698(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:44,661 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:44,661 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/B, priority=13, startTime=1731408824570; duration=0sec 2024-11-12T10:53:44,661 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:44,661 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:B 2024-11-12T10:53:44,661 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:44,663 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:44,663 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/C is initiating minor compaction (all files) 2024-11-12T10:53:44,663 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/C in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:44,663 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/2e924ed5445f4fd999e1853ffcd672fc, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/bb8e6d29b4b743d79a50763dcacafc7c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/0f54bcbb4ff8408d97dea2cbb15bbd33] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=35.3 K 2024-11-12T10:53:44,665 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e924ed5445f4fd999e1853ffcd672fc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731408820553 2024-11-12T10:53:44,670 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting bb8e6d29b4b743d79a50763dcacafc7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731408821213 2024-11-12T10:53:44,672 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f54bcbb4ff8408d97dea2cbb15bbd33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731408822447 2024-11-12T10:53:44,688 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#C#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:44,689 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/1a9f642212244c7da17a3f174db6a9b2 is 50, key is test_row_0/C:col10/1731408823617/Put/seqid=0 2024-11-12T10:53:44,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741857_1033 (size=12207) 2024-11-12T10:53:44,722 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/1a9f642212244c7da17a3f174db6a9b2 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/1a9f642212244c7da17a3f174db6a9b2 2024-11-12T10:53:44,738 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/C of b3cb1a6227d163d89478d6e9738e8055 into 1a9f642212244c7da17a3f174db6a9b2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:44,738 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:44,738 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/C, priority=13, startTime=1731408824571; duration=0sec 2024-11-12T10:53:44,739 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:44,739 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:C 2024-11-12T10:53:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:44,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-11-12T10:53:44,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:44,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:44,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:44,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:44,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:44,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:44,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/e0fb839f604049f49abbc6d7627f3a0c is 50, key is test_row_0/A:col10/1731408824818/Put/seqid=0 2024-11-12T10:53:44,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408884840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408884839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408884846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408884846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408884851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741858_1034 (size=12151) 2024-11-12T10:53:44,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408884958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408884958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408884958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408884958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:44,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:44,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408884967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:45,039 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/91cadca5c25d47339f787f2ebe2acd99 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/91cadca5c25d47339f787f2ebe2acd99 2024-11-12T10:53:45,049 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/A of b3cb1a6227d163d89478d6e9738e8055 into 91cadca5c25d47339f787f2ebe2acd99(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:45,049 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:45,049 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/A, priority=13, startTime=1731408824569; duration=0sec 2024-11-12T10:53:45,050 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:45,050 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:A 2024-11-12T10:53:45,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408885167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:45,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408885177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408885178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:45,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408885178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:45,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:45,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408885178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:45,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:45,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/e0fb839f604049f49abbc6d7627f3a0c 2024-11-12T10:53:45,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/ade96c19347b4171aaa878dd3f62efe4 is 50, key is test_row_0/B:col10/1731408824818/Put/seqid=0 2024-11-12T10:53:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T10:53:45,298 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:53:45,299 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:53:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-12T10:53:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T10:53:45,307 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:53:45,309 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:53:45,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:53:45,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741859_1035 (size=12151) 2024-11-12T10:53:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T10:53:45,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T10:53:45,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:45,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:45,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:45,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:45,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:45,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408885478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:45,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408885487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:45,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408885487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:45,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408885487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:45,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408885497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:45,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:45,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T10:53:45,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T10:53:45,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:45,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:45,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:45,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:45,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:45,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/ade96c19347b4171aaa878dd3f62efe4 2024-11-12T10:53:45,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/7fdc6b3a4f17472796efd2adad86998b is 50, key is test_row_0/C:col10/1731408824818/Put/seqid=0 2024-11-12T10:53:45,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741860_1036 (size=12151) 2024-11-12T10:53:45,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/7fdc6b3a4f17472796efd2adad86998b 2024-11-12T10:53:45,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/e0fb839f604049f49abbc6d7627f3a0c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/e0fb839f604049f49abbc6d7627f3a0c 2024-11-12T10:53:45,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/e0fb839f604049f49abbc6d7627f3a0c, entries=150, sequenceid=135, filesize=11.9 K 2024-11-12T10:53:45,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/ade96c19347b4171aaa878dd3f62efe4 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/ade96c19347b4171aaa878dd3f62efe4 2024-11-12T10:53:45,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/ade96c19347b4171aaa878dd3f62efe4, entries=150, sequenceid=135, filesize=11.9 K 2024-11-12T10:53:45,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/7fdc6b3a4f17472796efd2adad86998b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/7fdc6b3a4f17472796efd2adad86998b 2024-11-12T10:53:45,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T10:53:45,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:45,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:45,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:45,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:45,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:45,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/7fdc6b3a4f17472796efd2adad86998b, entries=150, sequenceid=135, filesize=11.9 K 2024-11-12T10:53:45,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=40.25 KB/41220 for b3cb1a6227d163d89478d6e9738e8055 in 966ms, sequenceid=135, compaction requested=false 2024-11-12T10:53:45,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T10:53:45,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T10:53:45,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:45,929 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-12T10:53:45,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:45,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:45,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:45,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:45,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:45,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:45,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/041781a22c354d8a989cf8d9ac4ace23 is 50, key is test_row_0/A:col10/1731408824840/Put/seqid=0 2024-11-12T10:53:45,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741861_1037 (size=12151) 2024-11-12T10:53:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:45,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:46,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731408826008, max timestamp=1731408826026Num uniques 859; 2024-11-12T10:53:46,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T10:53:46,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731408826008, max timestamp=1731408826026Num uniques 859; 2024-11-12T10:53:46,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731408826008, max timestamp=1731408826026Num uniques 859; 2024-11-12T10:53:46,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T10:53:46,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T10:53:46,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408886054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408886054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408886056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408886058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408886060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408886167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408886167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408886177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408886178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408886178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,345 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/041781a22c354d8a989cf8d9ac4ace23 2024-11-12T10:53:46,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/c70c6a742cb542a38060d9d1763b1d47 is 50, key is test_row_0/B:col10/1731408824840/Put/seqid=0 2024-11-12T10:53:46,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741862_1038 (size=12151) 2024-11-12T10:53:46,366 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/c70c6a742cb542a38060d9d1763b1d47 2024-11-12T10:53:46,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408886378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408886378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/232dcfffeb40479189cfc46f64073569 is 50, key is test_row_0/C:col10/1731408824840/Put/seqid=0 2024-11-12T10:53:46,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741863_1039 (size=12151) 2024-11-12T10:53:46,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408886398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408886398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408886398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T10:53:46,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408886687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408886688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408886708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408886708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408886708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:46,791 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/232dcfffeb40479189cfc46f64073569 2024-11-12T10:53:46,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/041781a22c354d8a989cf8d9ac4ace23 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/041781a22c354d8a989cf8d9ac4ace23 2024-11-12T10:53:46,815 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/041781a22c354d8a989cf8d9ac4ace23, entries=150, sequenceid=145, filesize=11.9 K 2024-11-12T10:53:46,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/c70c6a742cb542a38060d9d1763b1d47 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c70c6a742cb542a38060d9d1763b1d47 2024-11-12T10:53:46,825 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c70c6a742cb542a38060d9d1763b1d47, entries=150, sequenceid=145, filesize=11.9 K 2024-11-12T10:53:46,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/232dcfffeb40479189cfc46f64073569 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/232dcfffeb40479189cfc46f64073569 2024-11-12T10:53:46,838 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/232dcfffeb40479189cfc46f64073569, entries=150, sequenceid=145, filesize=11.9 K 2024-11-12T10:53:46,840 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for b3cb1a6227d163d89478d6e9738e8055 in 910ms, sequenceid=145, compaction requested=true 2024-11-12T10:53:46,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:46,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:46,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-12T10:53:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-12T10:53:46,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-12T10:53:46,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5330 sec 2024-11-12T10:53:46,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.5480 sec 2024-11-12T10:53:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:47,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-12T10:53:47,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:47,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:47,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:47,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:47,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:47,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:47,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/96b096c33d514da8b46aa6ccdb259094 is 50, key is test_row_0/A:col10/1731408826049/Put/seqid=0 2024-11-12T10:53:47,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408887223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408887225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408887225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408887229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408887229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741864_1040 (size=14541) 2024-11-12T10:53:47,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/96b096c33d514da8b46aa6ccdb259094 2024-11-12T10:53:47,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/1375775e70f241f2bac41548eb50a8dc is 50, key is test_row_0/B:col10/1731408826049/Put/seqid=0 2024-11-12T10:53:47,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741865_1041 (size=12151) 2024-11-12T10:53:47,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/1375775e70f241f2bac41548eb50a8dc 2024-11-12T10:53:47,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/f06d6b8cd52d42f8bfea900c00bfb913 is 50, key is test_row_0/C:col10/1731408826049/Put/seqid=0 2024-11-12T10:53:47,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741866_1042 (size=12151) 2024-11-12T10:53:47,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408887337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408887337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408887337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408887347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408887347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T10:53:47,448 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:53:47,448 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:53:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-12T10:53:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T10:53:47,451 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:53:47,452 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:53:47,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:53:47,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408887547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408887547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408887547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T10:53:47,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408887557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408887557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T10:53:47,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:47,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:47,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:47,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:47,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:47,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/f06d6b8cd52d42f8bfea900c00bfb913 2024-11-12T10:53:47,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/96b096c33d514da8b46aa6ccdb259094 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/96b096c33d514da8b46aa6ccdb259094 2024-11-12T10:53:47,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/96b096c33d514da8b46aa6ccdb259094, entries=200, sequenceid=182, filesize=14.2 K 2024-11-12T10:53:47,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/1375775e70f241f2bac41548eb50a8dc as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/1375775e70f241f2bac41548eb50a8dc 2024-11-12T10:53:47,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/1375775e70f241f2bac41548eb50a8dc, entries=150, sequenceid=182, filesize=11.9 K 2024-11-12T10:53:47,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/f06d6b8cd52d42f8bfea900c00bfb913 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/f06d6b8cd52d42f8bfea900c00bfb913 2024-11-12T10:53:47,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T10:53:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:47,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T10:53:47,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/f06d6b8cd52d42f8bfea900c00bfb913, entries=150, sequenceid=182, filesize=11.9 K 2024-11-12T10:53:47,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=40.25 KB/41220 for b3cb1a6227d163d89478d6e9738e8055 in 559ms, sequenceid=182, compaction requested=true 2024-11-12T10:53:47,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:47,771 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:53:47,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:53:47,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:47,773 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:53:47,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:53:47,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:47,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:53:47,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:47,777 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:53:47,777 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/B is initiating minor compaction (all files) 2024-11-12T10:53:47,777 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/B in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:47,777 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/61c9d781ae774bc3bccf98f736498698, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/ade96c19347b4171aaa878dd3f62efe4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c70c6a742cb542a38060d9d1763b1d47, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/1375775e70f241f2bac41548eb50a8dc] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=47.5 K 2024-11-12T10:53:47,778 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:53:47,779 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 61c9d781ae774bc3bccf98f736498698, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731408822447 2024-11-12T10:53:47,779 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/A is initiating minor compaction (all files) 2024-11-12T10:53:47,779 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/A in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:47,779 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/91cadca5c25d47339f787f2ebe2acd99, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/e0fb839f604049f49abbc6d7627f3a0c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/041781a22c354d8a989cf8d9ac4ace23, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/96b096c33d514da8b46aa6ccdb259094] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=49.9 K 2024-11-12T10:53:47,781 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91cadca5c25d47339f787f2ebe2acd99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731408822447 2024-11-12T10:53:47,781 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ade96c19347b4171aaa878dd3f62efe4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731408823673 2024-11-12T10:53:47,781 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0fb839f604049f49abbc6d7627f3a0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731408823673 2024-11-12T10:53:47,782 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting c70c6a742cb542a38060d9d1763b1d47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731408824839 2024-11-12T10:53:47,782 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 041781a22c354d8a989cf8d9ac4ace23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731408824839 2024-11-12T10:53:47,783 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 1375775e70f241f2bac41548eb50a8dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731408826049 2024-11-12T10:53:47,784 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b096c33d514da8b46aa6ccdb259094, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731408826049 2024-11-12T10:53:47,807 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#A#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:47,808 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/ade19b01cc9a4e42bb428d0ff3625619 is 50, key is test_row_0/A:col10/1731408826049/Put/seqid=0 2024-11-12T10:53:47,813 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:47,813 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/c916786713f148f4a4e6ee4b2a4c0f6f is 50, key is test_row_0/B:col10/1731408826049/Put/seqid=0 2024-11-12T10:53:47,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741867_1043 (size=12493) 2024-11-12T10:53:47,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741868_1044 (size=12493) 2024-11-12T10:53:47,834 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/ade19b01cc9a4e42bb428d0ff3625619 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ade19b01cc9a4e42bb428d0ff3625619 2024-11-12T10:53:47,846 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/A of b3cb1a6227d163d89478d6e9738e8055 into ade19b01cc9a4e42bb428d0ff3625619(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:47,846 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:47,846 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/A, priority=12, startTime=1731408827770; duration=0sec 2024-11-12T10:53:47,847 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:47,847 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:A 2024-11-12T10:53:47,847 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:53:47,854 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:53:47,854 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/C is initiating minor compaction (all files) 2024-11-12T10:53:47,854 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/C in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:47,855 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/1a9f642212244c7da17a3f174db6a9b2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/7fdc6b3a4f17472796efd2adad86998b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/232dcfffeb40479189cfc46f64073569, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/f06d6b8cd52d42f8bfea900c00bfb913] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=47.5 K 2024-11-12T10:53:47,855 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a9f642212244c7da17a3f174db6a9b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731408822447 2024-11-12T10:53:47,856 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fdc6b3a4f17472796efd2adad86998b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731408823673 2024-11-12T10:53:47,856 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 232dcfffeb40479189cfc46f64073569, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731408824839 2024-11-12T10:53:47,857 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting f06d6b8cd52d42f8bfea900c00bfb913, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731408826049 2024-11-12T10:53:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:47,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T10:53:47,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:47,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:47,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:47,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:47,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:47,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:47,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/be32388cded648148aef55b0cd01ec6f is 50, key is test_row_0/A:col10/1731408827858/Put/seqid=0 2024-11-12T10:53:47,883 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#C#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:47,884 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/9d4abd08ec614fc1a33e4d7861cc8ffb is 50, key is test_row_0/C:col10/1731408826049/Put/seqid=0 2024-11-12T10:53:47,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741869_1045 (size=12151) 2024-11-12T10:53:47,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/be32388cded648148aef55b0cd01ec6f 2024-11-12T10:53:47,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731408827875, max timestamp=1731408827902Num uniques 859; 2024-11-12T10:53:47,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731408827875, max timestamp=1731408827902Num uniques 859; 2024-11-12T10:53:47,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T10:53:47,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T10:53:47,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731408827875, max timestamp=1731408827902Num uniques 859; 2024-11-12T10:53:47,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T10:53:47,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/f39d49880c5946f1a325e1f9f7c40f00 is 50, key is test_row_0/B:col10/1731408827858/Put/seqid=0 2024-11-12T10:53:47,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741870_1046 (size=12493) 2024-11-12T10:53:47,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T10:53:47,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:47,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:47,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:47,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741871_1047 (size=12151) 2024-11-12T10:53:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:47,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408887927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408887929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408887929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408887938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:47,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408887938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408888047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408888047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408888047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408888048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408888057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T10:53:48,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:48,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:48,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:48,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T10:53:48,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T10:53:48,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:48,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:48,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:48,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,240 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/c916786713f148f4a4e6ee4b2a4c0f6f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c916786713f148f4a4e6ee4b2a4c0f6f 2024-11-12T10:53:48,247 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/B of b3cb1a6227d163d89478d6e9738e8055 into c916786713f148f4a4e6ee4b2a4c0f6f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:48,247 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:48,248 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/B, priority=12, startTime=1731408827773; duration=0sec 2024-11-12T10:53:48,248 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:48,248 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:B 2024-11-12T10:53:48,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408888257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408888257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408888257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408888257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408888267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,317 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/9d4abd08ec614fc1a33e4d7861cc8ffb as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9d4abd08ec614fc1a33e4d7861cc8ffb 2024-11-12T10:53:48,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/f39d49880c5946f1a325e1f9f7c40f00 2024-11-12T10:53:48,325 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/C of b3cb1a6227d163d89478d6e9738e8055 into 9d4abd08ec614fc1a33e4d7861cc8ffb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:48,325 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:48,325 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/C, priority=12, startTime=1731408827773; duration=0sec 2024-11-12T10:53:48,325 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:48,325 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:C 2024-11-12T10:53:48,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/5efd170583c64f9b9888704a714b6166 is 50, key is test_row_0/C:col10/1731408827858/Put/seqid=0 2024-11-12T10:53:48,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741872_1048 (size=12151) 2024-11-12T10:53:48,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T10:53:48,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:48,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:48,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:48,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T10:53:48,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:48,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:48,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:48,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408888567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408888567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408888577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408888577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408888577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T10:53:48,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T10:53:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:48,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:48,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/5efd170583c64f9b9888704a714b6166 2024-11-12T10:53:48,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/be32388cded648148aef55b0cd01ec6f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/be32388cded648148aef55b0cd01ec6f 2024-11-12T10:53:48,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/be32388cded648148aef55b0cd01ec6f, entries=150, sequenceid=193, filesize=11.9 K 2024-11-12T10:53:48,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/f39d49880c5946f1a325e1f9f7c40f00 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f39d49880c5946f1a325e1f9f7c40f00 2024-11-12T10:53:48,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f39d49880c5946f1a325e1f9f7c40f00, entries=150, sequenceid=193, filesize=11.9 K 2024-11-12T10:53:48,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/5efd170583c64f9b9888704a714b6166 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/5efd170583c64f9b9888704a714b6166 2024-11-12T10:53:48,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/5efd170583c64f9b9888704a714b6166, entries=150, sequenceid=193, filesize=11.9 K 2024-11-12T10:53:48,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for b3cb1a6227d163d89478d6e9738e8055 in 931ms, sequenceid=193, compaction requested=false 2024-11-12T10:53:48,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:48,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T10:53:48,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:48,839 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-12T10:53:48,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:48,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:48,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:48,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:48,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:48,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/b7d545ae206d4f02ab4aeae8cb7c5b53 is 50, key is test_row_0/A:col10/1731408827935/Put/seqid=0 2024-11-12T10:53:48,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741873_1049 (size=12151) 2024-11-12T10:53:48,856 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/b7d545ae206d4f02ab4aeae8cb7c5b53 2024-11-12T10:53:48,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/f4183ad64a5c44cc918a7ad3e6a6ec75 is 50, key is test_row_0/B:col10/1731408827935/Put/seqid=0 2024-11-12T10:53:48,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741874_1050 (size=12151) 2024-11-12T10:53:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:49,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:49,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408889092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408889093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408889093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408889094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408889098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408889207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408889207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408889207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408889207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408889207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,284 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/f4183ad64a5c44cc918a7ad3e6a6ec75 2024-11-12T10:53:49,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/6bafaee6953944a3b93d0e2942aae6f7 is 50, key is test_row_0/C:col10/1731408827935/Put/seqid=0 2024-11-12T10:53:49,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741875_1051 (size=12151) 2024-11-12T10:53:49,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408889417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408889418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408889418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408889418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408889418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T10:53:49,702 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/6bafaee6953944a3b93d0e2942aae6f7 2024-11-12T10:53:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/b7d545ae206d4f02ab4aeae8cb7c5b53 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/b7d545ae206d4f02ab4aeae8cb7c5b53 2024-11-12T10:53:49,715 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/b7d545ae206d4f02ab4aeae8cb7c5b53, entries=150, sequenceid=231, filesize=11.9 K 2024-11-12T10:53:49,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/f4183ad64a5c44cc918a7ad3e6a6ec75 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f4183ad64a5c44cc918a7ad3e6a6ec75 2024-11-12T10:53:49,723 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f4183ad64a5c44cc918a7ad3e6a6ec75, entries=150, sequenceid=231, filesize=11.9 K 2024-11-12T10:53:49,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/6bafaee6953944a3b93d0e2942aae6f7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6bafaee6953944a3b93d0e2942aae6f7 2024-11-12T10:53:49,732 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6bafaee6953944a3b93d0e2942aae6f7, entries=150, sequenceid=231, filesize=11.9 K 2024-11-12T10:53:49,733 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for b3cb1a6227d163d89478d6e9738e8055 in 894ms, sequenceid=231, compaction requested=true 2024-11-12T10:53:49,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:49,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:49,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-12T10:53:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-12T10:53:49,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T10:53:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:49,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:49,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:49,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:49,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:49,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:49,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:49,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-12T10:53:49,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2830 sec 2024-11-12T10:53:49,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/fc433261a23a4afaa8f2b2af149b4244 is 50, key is test_row_0/A:col10/1731408829736/Put/seqid=0 2024-11-12T10:53:49,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.2930 sec 2024-11-12T10:53:49,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741876_1052 (size=12151) 2024-11-12T10:53:49,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/fc433261a23a4afaa8f2b2af149b4244 2024-11-12T10:53:49,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/6116f37387034dfda0c5831e3d36a95a is 50, key is test_row_0/B:col10/1731408829736/Put/seqid=0 2024-11-12T10:53:49,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408889760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408889768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408889769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408889772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408889776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741877_1053 (size=12151) 2024-11-12T10:53:49,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/6116f37387034dfda0c5831e3d36a95a 2024-11-12T10:53:49,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/6eb998eb6b544021b36aa784d8a20231 is 50, key is test_row_0/C:col10/1731408829736/Put/seqid=0 2024-11-12T10:53:49,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741878_1054 (size=12151) 2024-11-12T10:53:49,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/6eb998eb6b544021b36aa784d8a20231 2024-11-12T10:53:49,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/fc433261a23a4afaa8f2b2af149b4244 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/fc433261a23a4afaa8f2b2af149b4244 2024-11-12T10:53:49,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/fc433261a23a4afaa8f2b2af149b4244, entries=150, sequenceid=243, filesize=11.9 K 2024-11-12T10:53:49,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/6116f37387034dfda0c5831e3d36a95a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/6116f37387034dfda0c5831e3d36a95a 2024-11-12T10:53:49,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/6116f37387034dfda0c5831e3d36a95a, entries=150, sequenceid=243, filesize=11.9 K 2024-11-12T10:53:49,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/6eb998eb6b544021b36aa784d8a20231 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6eb998eb6b544021b36aa784d8a20231 2024-11-12T10:53:49,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6eb998eb6b544021b36aa784d8a20231, entries=150, sequenceid=243, filesize=11.9 K 2024-11-12T10:53:49,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b3cb1a6227d163d89478d6e9738e8055 in 123ms, sequenceid=243, compaction requested=true 2024-11-12T10:53:49,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:49,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:53:49,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:49,859 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:53:49,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:53:49,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:49,859 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:53:49,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:53:49,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:49,865 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:53:49,865 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:53:49,865 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/A is initiating minor compaction (all files) 2024-11-12T10:53:49,865 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/B is initiating minor compaction (all files) 2024-11-12T10:53:49,865 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/B in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:49,865 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/A in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:49,865 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c916786713f148f4a4e6ee4b2a4c0f6f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f39d49880c5946f1a325e1f9f7c40f00, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f4183ad64a5c44cc918a7ad3e6a6ec75, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/6116f37387034dfda0c5831e3d36a95a] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=47.8 K 2024-11-12T10:53:49,865 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ade19b01cc9a4e42bb428d0ff3625619, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/be32388cded648148aef55b0cd01ec6f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/b7d545ae206d4f02ab4aeae8cb7c5b53, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/fc433261a23a4afaa8f2b2af149b4244] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=47.8 K 2024-11-12T10:53:49,866 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting c916786713f148f4a4e6ee4b2a4c0f6f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731408826049 2024-11-12T10:53:49,866 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting ade19b01cc9a4e42bb428d0ff3625619, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731408826049 2024-11-12T10:53:49,867 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting be32388cded648148aef55b0cd01ec6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731408827225 2024-11-12T10:53:49,867 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting f39d49880c5946f1a325e1f9f7c40f00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731408827225 2024-11-12T10:53:49,867 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7d545ae206d4f02ab4aeae8cb7c5b53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731408827927 2024-11-12T10:53:49,868 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting f4183ad64a5c44cc918a7ad3e6a6ec75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731408827927 2024-11-12T10:53:49,868 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc433261a23a4afaa8f2b2af149b4244, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731408829091 2024-11-12T10:53:49,868 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 6116f37387034dfda0c5831e3d36a95a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731408829091 2024-11-12T10:53:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:49,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-12T10:53:49,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:49,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:49,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:49,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:49,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:49,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:49,907 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#A#compaction#42 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:49,907 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:49,907 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/eab8a08bc3bf4357aca09800c72a03c3 is 50, key is test_row_0/A:col10/1731408829736/Put/seqid=0 2024-11-12T10:53:49,908 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/3221939eb2fa439da30b5868f550ece1 is 50, key is test_row_0/B:col10/1731408829736/Put/seqid=0 2024-11-12T10:53:49,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/44bb807c754e467c942ce93705c7b7de is 50, key is test_row_0/A:col10/1731408829891/Put/seqid=0 2024-11-12T10:53:49,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408889905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408889910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408889915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408889916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408889916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:49,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741879_1055 (size=12629) 2024-11-12T10:53:49,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741880_1056 (size=12629) 2024-11-12T10:53:49,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741881_1057 (size=14741) 2024-11-12T10:53:49,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/44bb807c754e467c942ce93705c7b7de 2024-11-12T10:53:49,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/accf259587d841dc8352570536488e29 is 50, key is test_row_0/B:col10/1731408829891/Put/seqid=0 2024-11-12T10:53:49,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741882_1058 (size=12301) 2024-11-12T10:53:50,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408890027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408890027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408890028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408890028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408890028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408890237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408890238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408890238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408890238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408890238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,345 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/3221939eb2fa439da30b5868f550ece1 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3221939eb2fa439da30b5868f550ece1 2024-11-12T10:53:50,347 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/eab8a08bc3bf4357aca09800c72a03c3 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/eab8a08bc3bf4357aca09800c72a03c3 2024-11-12T10:53:50,355 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/B of b3cb1a6227d163d89478d6e9738e8055 into 3221939eb2fa439da30b5868f550ece1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:50,355 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:50,355 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/B, priority=12, startTime=1731408829859; duration=0sec 2024-11-12T10:53:50,356 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:50,356 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:B 2024-11-12T10:53:50,356 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:53:50,357 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/A of b3cb1a6227d163d89478d6e9738e8055 into eab8a08bc3bf4357aca09800c72a03c3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:50,357 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:50,357 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/A, priority=12, startTime=1731408829859; duration=0sec 2024-11-12T10:53:50,357 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:50,357 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:A 2024-11-12T10:53:50,360 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:53:50,360 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/C is initiating minor compaction (all files) 2024-11-12T10:53:50,360 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/C in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:50,361 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9d4abd08ec614fc1a33e4d7861cc8ffb, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/5efd170583c64f9b9888704a714b6166, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6bafaee6953944a3b93d0e2942aae6f7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6eb998eb6b544021b36aa784d8a20231] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=47.8 K 2024-11-12T10:53:50,361 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d4abd08ec614fc1a33e4d7861cc8ffb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731408826049 2024-11-12T10:53:50,362 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 5efd170583c64f9b9888704a714b6166, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731408827225 2024-11-12T10:53:50,362 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bafaee6953944a3b93d0e2942aae6f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731408827927 2024-11-12T10:53:50,363 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb998eb6b544021b36aa784d8a20231, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731408829091 2024-11-12T10:53:50,379 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#C#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:50,380 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/9a7b1c42adf047b5923864026e73dc23 is 50, key is test_row_0/C:col10/1731408829736/Put/seqid=0 2024-11-12T10:53:50,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/accf259587d841dc8352570536488e29 2024-11-12T10:53:50,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741883_1059 (size=12629) 2024-11-12T10:53:50,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/318721ca80cd40e2b4d2255513ec9df9 is 50, key is test_row_0/C:col10/1731408829891/Put/seqid=0 2024-11-12T10:53:50,411 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/9a7b1c42adf047b5923864026e73dc23 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9a7b1c42adf047b5923864026e73dc23 2024-11-12T10:53:50,428 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/C of b3cb1a6227d163d89478d6e9738e8055 into 9a7b1c42adf047b5923864026e73dc23(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:50,428 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:50,429 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/C, priority=12, startTime=1731408829859; duration=0sec 2024-11-12T10:53:50,429 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:50,429 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:C 2024-11-12T10:53:50,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741884_1060 (size=12301) 2024-11-12T10:53:50,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408890547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408890548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408890548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408890557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:50,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408890557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:50,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/318721ca80cd40e2b4d2255513ec9df9 2024-11-12T10:53:50,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/44bb807c754e467c942ce93705c7b7de as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/44bb807c754e467c942ce93705c7b7de 2024-11-12T10:53:50,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/44bb807c754e467c942ce93705c7b7de, entries=200, sequenceid=265, filesize=14.4 K 2024-11-12T10:53:50,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/accf259587d841dc8352570536488e29 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/accf259587d841dc8352570536488e29 2024-11-12T10:53:50,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/accf259587d841dc8352570536488e29, entries=150, sequenceid=265, filesize=12.0 K 2024-11-12T10:53:50,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/318721ca80cd40e2b4d2255513ec9df9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/318721ca80cd40e2b4d2255513ec9df9 2024-11-12T10:53:50,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/318721ca80cd40e2b4d2255513ec9df9, entries=150, sequenceid=265, filesize=12.0 K 2024-11-12T10:53:50,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for b3cb1a6227d163d89478d6e9738e8055 in 974ms, sequenceid=265, compaction requested=false 2024-11-12T10:53:50,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:51,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T10:53:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:51,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/a0603fbf81bb41469c6d94e20fc77a3b is 50, key is test_row_0/A:col10/1731408829903/Put/seqid=0 2024-11-12T10:53:51,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741885_1061 (size=12301) 2024-11-12T10:53:51,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/a0603fbf81bb41469c6d94e20fc77a3b 2024-11-12T10:53:51,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/24749ccad05344e899ff0958b10173eb is 50, key is test_row_0/B:col10/1731408829903/Put/seqid=0 2024-11-12T10:53:51,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408891082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408891084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408891086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408891094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408891095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741886_1062 (size=12301) 2024-11-12T10:53:51,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408891197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408891207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408891207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408891208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408891208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408891406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408891417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408891417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408891427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408891427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/24749ccad05344e899ff0958b10173eb 2024-11-12T10:53:51,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/cd0a414aff714638aeaa9411ba754c90 is 50, key is test_row_0/C:col10/1731408829903/Put/seqid=0 2024-11-12T10:53:51,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741887_1063 (size=12301) 2024-11-12T10:53:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T10:53:51,607 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:53:51,608 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:53:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-12T10:53:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T10:53:51,610 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:53:51,611 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:53:51,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:53:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T10:53:51,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408891717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408891727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408891727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408891737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408891737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:51,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T10:53:51,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:51,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:51,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:51,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:51,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:51,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/cd0a414aff714638aeaa9411ba754c90 2024-11-12T10:53:51,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T10:53:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:51,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T10:53:51,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/a0603fbf81bb41469c6d94e20fc77a3b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/a0603fbf81bb41469c6d94e20fc77a3b 2024-11-12T10:53:51,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/a0603fbf81bb41469c6d94e20fc77a3b, entries=150, sequenceid=282, filesize=12.0 K 2024-11-12T10:53:51,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/24749ccad05344e899ff0958b10173eb as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/24749ccad05344e899ff0958b10173eb 2024-11-12T10:53:51,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/24749ccad05344e899ff0958b10173eb, entries=150, sequenceid=282, filesize=12.0 K 2024-11-12T10:53:51,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/cd0a414aff714638aeaa9411ba754c90 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/cd0a414aff714638aeaa9411ba754c90 2024-11-12T10:53:51,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/cd0a414aff714638aeaa9411ba754c90, entries=150, sequenceid=282, filesize=12.0 K 2024-11-12T10:53:51,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b3cb1a6227d163d89478d6e9738e8055 in 889ms, sequenceid=282, compaction requested=true 2024-11-12T10:53:51,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:51,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:53:51,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:51,950 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:51,950 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:51,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:53:51,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:51,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:53:51,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:51,951 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:51,951 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:51,951 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/B is initiating minor compaction (all files) 2024-11-12T10:53:51,952 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/A is initiating minor compaction (all files) 2024-11-12T10:53:51,952 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/B in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:51,952 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/A in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:51,952 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/eab8a08bc3bf4357aca09800c72a03c3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/44bb807c754e467c942ce93705c7b7de, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/a0603fbf81bb41469c6d94e20fc77a3b] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=38.7 K 2024-11-12T10:53:51,952 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3221939eb2fa439da30b5868f550ece1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/accf259587d841dc8352570536488e29, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/24749ccad05344e899ff0958b10173eb] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=36.4 K 2024-11-12T10:53:51,952 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 3221939eb2fa439da30b5868f550ece1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731408829091 2024-11-12T10:53:51,952 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting eab8a08bc3bf4357aca09800c72a03c3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731408829091 2024-11-12T10:53:51,953 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting accf259587d841dc8352570536488e29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731408829767 2024-11-12T10:53:51,953 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44bb807c754e467c942ce93705c7b7de, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731408829767 2024-11-12T10:53:51,953 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 24749ccad05344e899ff0958b10173eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731408829903 2024-11-12T10:53:51,953 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0603fbf81bb41469c6d94e20fc77a3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731408829903 2024-11-12T10:53:51,964 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:51,965 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/3cf2c477fc9740d4a5b48b2064a4fd64 is 50, key is test_row_0/B:col10/1731408829903/Put/seqid=0 2024-11-12T10:53:51,968 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#A#compaction#51 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:51,969 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/b57cc4398e6444bc8d19652182bf0fbb is 50, key is test_row_0/A:col10/1731408829903/Put/seqid=0 2024-11-12T10:53:51,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741888_1064 (size=12881) 2024-11-12T10:53:51,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741889_1065 (size=12881) 2024-11-12T10:53:52,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T10:53:52,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:52,074 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T10:53:52,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:52,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:52,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:52,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:52,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:52,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:52,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/78c24e93773c43378b505ddfa7a533ba is 50, key is test_row_0/A:col10/1731408831091/Put/seqid=0 2024-11-12T10:53:52,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741890_1066 (size=12301) 2024-11-12T10:53:52,088 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/78c24e93773c43378b505ddfa7a533ba 2024-11-12T10:53:52,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/fcc1b73f8b0047a9aa48b91d36fb3a86 is 50, key is test_row_0/B:col10/1731408831091/Put/seqid=0 2024-11-12T10:53:52,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741891_1067 (size=12301) 2024-11-12T10:53:52,111 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/fcc1b73f8b0047a9aa48b91d36fb3a86 2024-11-12T10:53:52,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/8dac532863af40f5bb3e79154da0031c is 50, key is test_row_0/C:col10/1731408831091/Put/seqid=0 2024-11-12T10:53:52,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741892_1068 (size=12301) 2024-11-12T10:53:52,148 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/8dac532863af40f5bb3e79154da0031c 2024-11-12T10:53:52,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/78c24e93773c43378b505ddfa7a533ba as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/78c24e93773c43378b505ddfa7a533ba 2024-11-12T10:53:52,163 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/78c24e93773c43378b505ddfa7a533ba, entries=150, sequenceid=300, filesize=12.0 K 2024-11-12T10:53:52,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/fcc1b73f8b0047a9aa48b91d36fb3a86 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/fcc1b73f8b0047a9aa48b91d36fb3a86 2024-11-12T10:53:52,176 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/fcc1b73f8b0047a9aa48b91d36fb3a86, entries=150, sequenceid=300, filesize=12.0 K 2024-11-12T10:53:52,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/8dac532863af40f5bb3e79154da0031c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8dac532863af40f5bb3e79154da0031c 2024-11-12T10:53:52,189 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8dac532863af40f5bb3e79154da0031c, entries=150, sequenceid=300, filesize=12.0 K 2024-11-12T10:53:52,191 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for b3cb1a6227d163d89478d6e9738e8055 in 117ms, sequenceid=300, compaction requested=true 2024-11-12T10:53:52,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:52,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:52,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-12T10:53:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-12T10:53:52,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-12T10:53:52,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 582 msec 2024-11-12T10:53:52,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 589 msec 2024-11-12T10:53:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T10:53:52,238 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:53:52,238 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:53:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-12T10:53:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T10:53:52,241 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:53:52,243 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:53:52,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:53:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:52,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T10:53:52,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:52,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:52,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:52,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:52,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:52,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:52,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/f7bd626ea4db44249341389dfafc953b is 50, key is test_row_0/A:col10/1731408832243/Put/seqid=0 2024-11-12T10:53:52,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741893_1069 (size=17181) 2024-11-12T10:53:52,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/f7bd626ea4db44249341389dfafc953b 2024-11-12T10:53:52,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1731408832274, max timestamp=1731408832302Num uniques 859; 2024-11-12T10:53:52,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-12T10:53:52,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1731408832274, max timestamp=1731408832302Num uniques 859; 2024-11-12T10:53:52,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-12T10:53:52,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1731408832274, max timestamp=1731408832302Num uniques 859; 2024-11-12T10:53:52,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-12T10:53:52,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/68c96eadc8fa4f0f84479082b909b60d is 50, key is test_row_0/B:col10/1731408832243/Put/seqid=0 2024-11-12T10:53:52,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741894_1070 (size=12301) 2024-11-12T10:53:52,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/68c96eadc8fa4f0f84479082b909b60d 2024-11-12T10:53:52,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408892327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408892328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408892330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408892332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/77e8b075ab2c44eabe4528f2b37c0f83 is 50, key is test_row_0/C:col10/1731408832243/Put/seqid=0 2024-11-12T10:53:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T10:53:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741895_1071 (size=12301) 2024-11-12T10:53:52,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/77e8b075ab2c44eabe4528f2b37c0f83 2024-11-12T10:53:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408892344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/f7bd626ea4db44249341389dfafc953b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/f7bd626ea4db44249341389dfafc953b 2024-11-12T10:53:52,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/f7bd626ea4db44249341389dfafc953b, entries=250, sequenceid=310, filesize=16.8 K 2024-11-12T10:53:52,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/68c96eadc8fa4f0f84479082b909b60d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/68c96eadc8fa4f0f84479082b909b60d 2024-11-12T10:53:52,385 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/b57cc4398e6444bc8d19652182bf0fbb as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/b57cc4398e6444bc8d19652182bf0fbb 2024-11-12T10:53:52,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/68c96eadc8fa4f0f84479082b909b60d, entries=150, sequenceid=310, filesize=12.0 K 2024-11-12T10:53:52,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/77e8b075ab2c44eabe4528f2b37c0f83 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/77e8b075ab2c44eabe4528f2b37c0f83 2024-11-12T10:53:52,395 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/A of b3cb1a6227d163d89478d6e9738e8055 into b57cc4398e6444bc8d19652182bf0fbb(size=12.6 K), total size for store is 41.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:52,395 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:52,395 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/A, priority=13, startTime=1731408831950; duration=0sec 2024-11-12T10:53:52,395 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:52,395 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:A 2024-11-12T10:53:52,395 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:53:52,402 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:53:52,402 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/C is initiating minor compaction (all files) 2024-11-12T10:53:52,402 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/C in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:52,402 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9a7b1c42adf047b5923864026e73dc23, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/318721ca80cd40e2b4d2255513ec9df9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/cd0a414aff714638aeaa9411ba754c90, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8dac532863af40f5bb3e79154da0031c] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=48.4 K 2024-11-12T10:53:52,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T10:53:52,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:52,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:52,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:52,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:52,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:52,404 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a7b1c42adf047b5923864026e73dc23, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731408829091 2024-11-12T10:53:52,405 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 318721ca80cd40e2b4d2255513ec9df9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731408829767 2024-11-12T10:53:52,406 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd0a414aff714638aeaa9411ba754c90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731408829903 2024-11-12T10:53:52,406 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dac532863af40f5bb3e79154da0031c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731408831078 2024-11-12T10:53:52,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/77e8b075ab2c44eabe4528f2b37c0f83, entries=150, sequenceid=310, filesize=12.0 K 2024-11-12T10:53:52,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for b3cb1a6227d163d89478d6e9738e8055 in 153ms, sequenceid=310, compaction requested=false 2024-11-12T10:53:52,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:52,419 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/3cf2c477fc9740d4a5b48b2064a4fd64 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3cf2c477fc9740d4a5b48b2064a4fd64 2024-11-12T10:53:52,435 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/B of b3cb1a6227d163d89478d6e9738e8055 into 3cf2c477fc9740d4a5b48b2064a4fd64(size=12.6 K), total size for store is 36.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:52,435 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:52,435 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/B, priority=13, startTime=1731408831950; duration=0sec 2024-11-12T10:53:52,435 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:52,435 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:B 2024-11-12T10:53:52,436 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:52,437 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/2f23e8b890c74735b8d6ee67b2d0dac5 is 50, key is test_row_0/C:col10/1731408831091/Put/seqid=0 2024-11-12T10:53:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:52,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=147.60 KB heapSize=392.02 KB 2024-11-12T10:53:52,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:52,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:52,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:52,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:52,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:52,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:53:52,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741896_1072 (size=12915) 2024-11-12T10:53:52,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/5f948eccd8c94b50b921ab9b36a2bceb is 50, key is test_row_0/A:col10/1731408832330/Put/seqid=0 2024-11-12T10:53:52,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408892469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408892470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408892472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408892478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408892479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,482 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/2f23e8b890c74735b8d6ee67b2d0dac5 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/2f23e8b890c74735b8d6ee67b2d0dac5 2024-11-12T10:53:52,491 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/C of b3cb1a6227d163d89478d6e9738e8055 into 2f23e8b890c74735b8d6ee67b2d0dac5(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:52,491 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:52,491 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/C, priority=12, startTime=1731408831950; duration=0sec 2024-11-12T10:53:52,492 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:52,492 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:C 2024-11-12T10:53:52,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741897_1073 (size=14741) 2024-11-12T10:53:52,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T10:53:52,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T10:53:52,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:52,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:52,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:52,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:52,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408892587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408892588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408892588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408892588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408892589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T10:53:52,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:52,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:52,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:52,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:52,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:52,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408892798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408892798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408892799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408892799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408892799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T10:53:52,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T10:53:52,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:52,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:52,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:52,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:52,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:52,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/5f948eccd8c94b50b921ab9b36a2bceb 2024-11-12T10:53:52,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/4d652dd3fdba48a995230f01381d0953 is 50, key is test_row_0/B:col10/1731408832330/Put/seqid=0 2024-11-12T10:53:52,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741898_1074 (size=12301) 2024-11-12T10:53:53,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T10:53:53,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:53,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:53,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:53,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408893117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:53,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:53,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408893117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:53,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:53,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:53,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408893117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408893118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408893118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:53,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T10:53:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:53,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/4d652dd3fdba48a995230f01381d0953 2024-11-12T10:53:53,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/ad129db58b564a0e92f02a5ddceffb9c is 50, key is test_row_0/C:col10/1731408832330/Put/seqid=0 2024-11-12T10:53:53,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741899_1075 (size=12301) 2024-11-12T10:53:53,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T10:53:53,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:53,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:53,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:53,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T10:53:53,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T10:53:53,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:53,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:53,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:53,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:53,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:53,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408893627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:53,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408893627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:53,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408893628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:53,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408893627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:53,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408893628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:53,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T10:53:53,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:53,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:53,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:53,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:53,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/ad129db58b564a0e92f02a5ddceffb9c 2024-11-12T10:53:53,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/5f948eccd8c94b50b921ab9b36a2bceb as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/5f948eccd8c94b50b921ab9b36a2bceb 2024-11-12T10:53:53,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/5f948eccd8c94b50b921ab9b36a2bceb, entries=200, sequenceid=344, filesize=14.4 K 2024-11-12T10:53:53,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/4d652dd3fdba48a995230f01381d0953 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/4d652dd3fdba48a995230f01381d0953 2024-11-12T10:53:53,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/4d652dd3fdba48a995230f01381d0953, entries=150, sequenceid=344, filesize=12.0 K 2024-11-12T10:53:53,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/ad129db58b564a0e92f02a5ddceffb9c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/ad129db58b564a0e92f02a5ddceffb9c 2024-11-12T10:53:53,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/ad129db58b564a0e92f02a5ddceffb9c, entries=150, sequenceid=344, filesize=12.0 K 2024-11-12T10:53:53,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~391.97 KB/401376, currentSize=46.96 KB/48090 for b3cb1a6227d163d89478d6e9738e8055 in 1306ms, sequenceid=344, compaction requested=true 2024-11-12T10:53:53,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:53,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:53:53,755 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:53:53,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:53,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:53:53,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:53,755 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:53:53,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:53:53,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:53,757 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:53:53,757 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57104 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:53:53,757 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/B is initiating minor compaction (all files) 2024-11-12T10:53:53,757 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/A is initiating minor compaction (all files) 2024-11-12T10:53:53,757 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/B in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:53,757 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/A in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:53,757 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3cf2c477fc9740d4a5b48b2064a4fd64, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/fcc1b73f8b0047a9aa48b91d36fb3a86, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/68c96eadc8fa4f0f84479082b909b60d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/4d652dd3fdba48a995230f01381d0953] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=48.6 K 2024-11-12T10:53:53,757 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/b57cc4398e6444bc8d19652182bf0fbb, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/78c24e93773c43378b505ddfa7a533ba, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/f7bd626ea4db44249341389dfafc953b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/5f948eccd8c94b50b921ab9b36a2bceb] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=55.8 K 2024-11-12T10:53:53,757 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cf2c477fc9740d4a5b48b2064a4fd64, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731408829903 2024-11-12T10:53:53,757 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting b57cc4398e6444bc8d19652182bf0fbb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731408829903 2024-11-12T10:53:53,758 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting fcc1b73f8b0047a9aa48b91d36fb3a86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731408831078 2024-11-12T10:53:53,758 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78c24e93773c43378b505ddfa7a533ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731408831078 2024-11-12T10:53:53,758 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 68c96eadc8fa4f0f84479082b909b60d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731408832243 2024-11-12T10:53:53,758 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7bd626ea4db44249341389dfafc953b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731408832239 2024-11-12T10:53:53,758 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d652dd3fdba48a995230f01381d0953, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731408832330 2024-11-12T10:53:53,758 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f948eccd8c94b50b921ab9b36a2bceb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731408832321 2024-11-12T10:53:53,769 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#A#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:53,770 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/23864e6dc08847b282e1aa5a382f0a56 is 50, key is test_row_0/A:col10/1731408832330/Put/seqid=0 2024-11-12T10:53:53,778 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:53,779 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/57feb8d3ce4346bbb9eaa63e9d544dce is 50, key is test_row_0/B:col10/1731408832330/Put/seqid=0 2024-11-12T10:53:53,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741901_1077 (size=13017) 2024-11-12T10:53:53,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741900_1076 (size=13017) 2024-11-12T10:53:53,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T10:53:53,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:53,790 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T10:53:53,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:53,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:53,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:53,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:53,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:53,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:53,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/432a8791d6b14b8da93f75edeeb69077 is 50, key is test_row_0/A:col10/1731408832475/Put/seqid=0 2024-11-12T10:53:53,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741902_1078 (size=12301) 2024-11-12T10:53:53,806 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/432a8791d6b14b8da93f75edeeb69077 2024-11-12T10:53:53,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/2f4048ce3af143db942969b7875028e1 is 50, key is test_row_0/B:col10/1731408832475/Put/seqid=0 2024-11-12T10:53:53,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741903_1079 (size=12301) 2024-11-12T10:53:54,213 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/23864e6dc08847b282e1aa5a382f0a56 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/23864e6dc08847b282e1aa5a382f0a56 2024-11-12T10:53:54,213 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/57feb8d3ce4346bbb9eaa63e9d544dce as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/57feb8d3ce4346bbb9eaa63e9d544dce 2024-11-12T10:53:54,223 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/B of b3cb1a6227d163d89478d6e9738e8055 into 57feb8d3ce4346bbb9eaa63e9d544dce(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:54,223 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:54,223 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/B, priority=12, startTime=1731408833755; duration=0sec 2024-11-12T10:53:54,223 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:54,223 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:B 2024-11-12T10:53:54,223 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:54,224 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/A of b3cb1a6227d163d89478d6e9738e8055 into 23864e6dc08847b282e1aa5a382f0a56(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:54,224 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:54,224 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/A, priority=12, startTime=1731408833755; duration=0sec 2024-11-12T10:53:54,224 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:54,224 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:A 2024-11-12T10:53:54,225 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:54,225 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/C is initiating minor compaction (all files) 2024-11-12T10:53:54,225 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/C in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:54,226 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/2f23e8b890c74735b8d6ee67b2d0dac5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/77e8b075ab2c44eabe4528f2b37c0f83, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/ad129db58b564a0e92f02a5ddceffb9c] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=36.6 K 2024-11-12T10:53:54,227 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f23e8b890c74735b8d6ee67b2d0dac5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731408831078 2024-11-12T10:53:54,228 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 77e8b075ab2c44eabe4528f2b37c0f83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731408832243 2024-11-12T10:53:54,228 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/2f4048ce3af143db942969b7875028e1 2024-11-12T10:53:54,228 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ad129db58b564a0e92f02a5ddceffb9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731408832330 2024-11-12T10:53:54,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/8ff00deabe644c678ade523d1d320493 is 50, key is test_row_0/C:col10/1731408832475/Put/seqid=0 2024-11-12T10:53:54,245 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:54,246 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/e0e531500a68413f8ac691130ec7049a is 50, key is test_row_0/C:col10/1731408832330/Put/seqid=0 2024-11-12T10:53:54,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741904_1080 (size=12301) 2024-11-12T10:53:54,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741905_1081 (size=13017) 2024-11-12T10:53:54,262 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/e0e531500a68413f8ac691130ec7049a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/e0e531500a68413f8ac691130ec7049a 2024-11-12T10:53:54,274 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/C of b3cb1a6227d163d89478d6e9738e8055 into e0e531500a68413f8ac691130ec7049a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:54,274 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:54,274 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/C, priority=13, startTime=1731408833755; duration=0sec 2024-11-12T10:53:54,274 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:54,274 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:C 2024-11-12T10:53:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T10:53:54,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:54,656 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/8ff00deabe644c678ade523d1d320493 2024-11-12T10:53:54,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/432a8791d6b14b8da93f75edeeb69077 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/432a8791d6b14b8da93f75edeeb69077 2024-11-12T10:53:54,667 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/432a8791d6b14b8da93f75edeeb69077, entries=150, sequenceid=355, filesize=12.0 K 2024-11-12T10:53:54,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/2f4048ce3af143db942969b7875028e1 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/2f4048ce3af143db942969b7875028e1 2024-11-12T10:53:54,673 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/2f4048ce3af143db942969b7875028e1, entries=150, sequenceid=355, filesize=12.0 K 2024-11-12T10:53:54,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/8ff00deabe644c678ade523d1d320493 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8ff00deabe644c678ade523d1d320493 2024-11-12T10:53:54,681 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8ff00deabe644c678ade523d1d320493, entries=150, sequenceid=355, filesize=12.0 K 2024-11-12T10:53:54,683 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=73.80 KB/75570 for b3cb1a6227d163d89478d6e9738e8055 in 892ms, sequenceid=355, compaction requested=false 2024-11-12T10:53:54,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:54,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:54,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-12T10:53:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-12T10:53:54,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-12T10:53:54,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4410 sec 2024-11-12T10:53:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:54,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T10:53:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:54,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.4490 sec 2024-11-12T10:53:54,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/4de9a153e39442d680274806a261a5dc is 50, key is test_row_0/A:col10/1731408834676/Put/seqid=0 2024-11-12T10:53:54,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741906_1082 (size=14741) 2024-11-12T10:53:54,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/4de9a153e39442d680274806a261a5dc 2024-11-12T10:53:54,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408894704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:54,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408894711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:54,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408894713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:54,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408894714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:54,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408894714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:54,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/3642bfe6b04b41de96921c6e493dffb3 is 50, key is test_row_0/B:col10/1731408834676/Put/seqid=0 2024-11-12T10:53:54,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741907_1083 (size=12301) 2024-11-12T10:53:54,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408894827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:54,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:54,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408894827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:54,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:54,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:54,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408894827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:54,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:54,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408894827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408894827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408895037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408895037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408895037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408895038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408895038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/3642bfe6b04b41de96921c6e493dffb3 2024-11-12T10:53:55,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/05cb465732c34a7d9ae40125bbba58f4 is 50, key is test_row_0/C:col10/1731408834676/Put/seqid=0 2024-11-12T10:53:55,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741908_1084 (size=12301) 2024-11-12T10:53:55,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408895347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408895347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408895348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408895348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408895348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/05cb465732c34a7d9ae40125bbba58f4 2024-11-12T10:53:55,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/4de9a153e39442d680274806a261a5dc as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/4de9a153e39442d680274806a261a5dc 2024-11-12T10:53:55,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/4de9a153e39442d680274806a261a5dc, entries=200, sequenceid=373, filesize=14.4 K 2024-11-12T10:53:55,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/3642bfe6b04b41de96921c6e493dffb3 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3642bfe6b04b41de96921c6e493dffb3 2024-11-12T10:53:55,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3642bfe6b04b41de96921c6e493dffb3, entries=150, sequenceid=373, filesize=12.0 K 2024-11-12T10:53:55,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/05cb465732c34a7d9ae40125bbba58f4 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/05cb465732c34a7d9ae40125bbba58f4 2024-11-12T10:53:55,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/05cb465732c34a7d9ae40125bbba58f4, entries=150, sequenceid=373, filesize=12.0 K 2024-11-12T10:53:55,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b3cb1a6227d163d89478d6e9738e8055 in 894ms, sequenceid=373, compaction requested=true 2024-11-12T10:53:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:53:55,582 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:53:55,582 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:53:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:55,583 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:55,583 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:55,583 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/A is initiating minor compaction (all files) 2024-11-12T10:53:55,583 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/B is initiating minor compaction (all files) 2024-11-12T10:53:55,583 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/A in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:55,583 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/B in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:55,583 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/23864e6dc08847b282e1aa5a382f0a56, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/432a8791d6b14b8da93f75edeeb69077, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/4de9a153e39442d680274806a261a5dc] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=39.1 K 2024-11-12T10:53:55,583 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/57feb8d3ce4346bbb9eaa63e9d544dce, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/2f4048ce3af143db942969b7875028e1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3642bfe6b04b41de96921c6e493dffb3] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=36.7 K 2024-11-12T10:53:55,584 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 57feb8d3ce4346bbb9eaa63e9d544dce, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731408832330 2024-11-12T10:53:55,584 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23864e6dc08847b282e1aa5a382f0a56, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731408832330 2024-11-12T10:53:55,584 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f4048ce3af143db942969b7875028e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731408832467 2024-11-12T10:53:55,584 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 432a8791d6b14b8da93f75edeeb69077, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731408832467 2024-11-12T10:53:55,584 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 3642bfe6b04b41de96921c6e493dffb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731408834676 2024-11-12T10:53:55,584 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4de9a153e39442d680274806a261a5dc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731408834646 2024-11-12T10:53:55,595 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:55,596 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/6ce5659b2f484ae5871bffc912dd57db is 50, key is test_row_0/A:col10/1731408834676/Put/seqid=0 2024-11-12T10:53:55,600 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#B#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:55,601 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/3fa6338f9288426b8211340be2f59e15 is 50, key is test_row_0/B:col10/1731408834676/Put/seqid=0 2024-11-12T10:53:55,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741910_1086 (size=13119) 2024-11-12T10:53:55,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741909_1085 (size=13119) 2024-11-12T10:53:55,619 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/6ce5659b2f484ae5871bffc912dd57db as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/6ce5659b2f484ae5871bffc912dd57db 2024-11-12T10:53:55,626 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/A of b3cb1a6227d163d89478d6e9738e8055 into 6ce5659b2f484ae5871bffc912dd57db(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:55,626 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:55,626 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/A, priority=13, startTime=1731408835582; duration=0sec 2024-11-12T10:53:55,626 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:55,626 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:A 2024-11-12T10:53:55,626 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:55,627 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:55,628 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/C is initiating minor compaction (all files) 2024-11-12T10:53:55,628 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/C in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:55,628 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/e0e531500a68413f8ac691130ec7049a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8ff00deabe644c678ade523d1d320493, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/05cb465732c34a7d9ae40125bbba58f4] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=36.7 K 2024-11-12T10:53:55,628 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0e531500a68413f8ac691130ec7049a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731408832330 2024-11-12T10:53:55,629 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ff00deabe644c678ade523d1d320493, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731408832467 2024-11-12T10:53:55,629 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05cb465732c34a7d9ae40125bbba58f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731408834676 2024-11-12T10:53:55,637 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#C#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:55,637 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/6c207a0ca5fd44388c3c08012e4c7103 is 50, key is test_row_0/C:col10/1731408834676/Put/seqid=0 2024-11-12T10:53:55,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741911_1087 (size=13119) 2024-11-12T10:53:55,652 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/6c207a0ca5fd44388c3c08012e4c7103 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6c207a0ca5fd44388c3c08012e4c7103 2024-11-12T10:53:55,659 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/C of b3cb1a6227d163d89478d6e9738e8055 into 6c207a0ca5fd44388c3c08012e4c7103(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:55,659 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:55,659 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/C, priority=13, startTime=1731408835582; duration=0sec 2024-11-12T10:53:55,659 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:55,659 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:C 2024-11-12T10:53:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:55,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T10:53:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:55,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/1b0a5e20d0194714af5327b0a7b64409 is 50, key is test_row_0/A:col10/1731408834703/Put/seqid=0 2024-11-12T10:53:55,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408895868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408895869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408895870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408895874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408895874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741912_1088 (size=17181) 2024-11-12T10:53:55,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/1b0a5e20d0194714af5327b0a7b64409 2024-11-12T10:53:55,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/c52ef74a433b4f6aae853109eca2d9d1 is 50, key is test_row_0/B:col10/1731408834703/Put/seqid=0 2024-11-12T10:53:55,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741913_1089 (size=12301) 2024-11-12T10:53:55,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/c52ef74a433b4f6aae853109eca2d9d1 2024-11-12T10:53:55,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/dc2e79b4c02b46ecb8aae2462213e26a is 50, key is test_row_0/C:col10/1731408834703/Put/seqid=0 2024-11-12T10:53:55,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741914_1090 (size=12301) 2024-11-12T10:53:55,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408895977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408895977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408895977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408895977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:55,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408895987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,017 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/3fa6338f9288426b8211340be2f59e15 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3fa6338f9288426b8211340be2f59e15 2024-11-12T10:53:56,024 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/B of b3cb1a6227d163d89478d6e9738e8055 into 3fa6338f9288426b8211340be2f59e15(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:56,024 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:56,024 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/B, priority=13, startTime=1731408835582; duration=0sec 2024-11-12T10:53:56,024 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:56,024 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:B 2024-11-12T10:53:56,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408896187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408896187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408896187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408896188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408896197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/dc2e79b4c02b46ecb8aae2462213e26a 2024-11-12T10:53:56,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/1b0a5e20d0194714af5327b0a7b64409 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/1b0a5e20d0194714af5327b0a7b64409 2024-11-12T10:53:56,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/1b0a5e20d0194714af5327b0a7b64409, entries=250, sequenceid=394, filesize=16.8 K 2024-11-12T10:53:56,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/c52ef74a433b4f6aae853109eca2d9d1 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c52ef74a433b4f6aae853109eca2d9d1 2024-11-12T10:53:56,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c52ef74a433b4f6aae853109eca2d9d1, entries=150, sequenceid=394, filesize=12.0 K 2024-11-12T10:53:56,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/dc2e79b4c02b46ecb8aae2462213e26a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/dc2e79b4c02b46ecb8aae2462213e26a 2024-11-12T10:53:56,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/dc2e79b4c02b46ecb8aae2462213e26a, entries=150, sequenceid=394, filesize=12.0 K 2024-11-12T10:53:56,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b3cb1a6227d163d89478d6e9738e8055 in 509ms, sequenceid=394, compaction requested=false 2024-11-12T10:53:56,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T10:53:56,397 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:53:56,398 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:53:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-12T10:53:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T10:53:56,400 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:53:56,401 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:53:56,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:53:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:56,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T10:53:56,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:56,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:56,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:56,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:56,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:56,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:56,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/c1abb3b11325432c9be2457b7914d31e is 50, key is test_row_0/A:col10/1731408835868/Put/seqid=0 2024-11-12T10:53:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T10:53:56,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408896517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408896517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408896519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408896521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408896522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741915_1091 (size=12301) 2024-11-12T10:53:56,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T10:53:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:56,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:56,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408896627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408896627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408896637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408896637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T10:53:56,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:56,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:56,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:56,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:56,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T10:53:56,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408896837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408896837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408896847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408896847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:56,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T10:53:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:56,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:56,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/c1abb3b11325432c9be2457b7914d31e 2024-11-12T10:53:57,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T10:53:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:57,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T10:53:57,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/6120fc31021f4805aae0d19824b04f14 is 50, key is test_row_0/B:col10/1731408835868/Put/seqid=0 2024-11-12T10:53:57,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741916_1092 (size=12301) 2024-11-12T10:53:57,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408897037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:57,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408897147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:57,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408897147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:57,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408897157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:57,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408897157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:57,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T10:53:57,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:57,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T10:53:57,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:57,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/6120fc31021f4805aae0d19824b04f14 2024-11-12T10:53:57,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/9ea63c9f00b2497fbbfd22a4bbb2d473 is 50, key is test_row_0/C:col10/1731408835868/Put/seqid=0 2024-11-12T10:53:57,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741917_1093 (size=12301) 2024-11-12T10:53:57,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T10:53:57,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:57,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T10:53:57,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T10:53:57,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:57,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408897657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408897657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:57,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:57,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408897687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:57,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408897687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:57,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T10:53:57,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:57,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:57,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/9ea63c9f00b2497fbbfd22a4bbb2d473 2024-11-12T10:53:57,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/c1abb3b11325432c9be2457b7914d31e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/c1abb3b11325432c9be2457b7914d31e 2024-11-12T10:53:57,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/c1abb3b11325432c9be2457b7914d31e, entries=150, sequenceid=409, filesize=12.0 K 2024-11-12T10:53:57,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/6120fc31021f4805aae0d19824b04f14 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/6120fc31021f4805aae0d19824b04f14 2024-11-12T10:53:57,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/6120fc31021f4805aae0d19824b04f14, entries=150, sequenceid=409, filesize=12.0 K 2024-11-12T10:53:57,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/9ea63c9f00b2497fbbfd22a4bbb2d473 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9ea63c9f00b2497fbbfd22a4bbb2d473 2024-11-12T10:53:57,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9ea63c9f00b2497fbbfd22a4bbb2d473, entries=150, sequenceid=409, filesize=12.0 K 2024-11-12T10:53:57,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b3cb1a6227d163d89478d6e9738e8055 in 1381ms, sequenceid=409, compaction requested=true 2024-11-12T10:53:57,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:57,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:53:57,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:57,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:53:57,881 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:57,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:57,881 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:57,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:53:57,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:57,882 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:57,882 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/B is initiating minor compaction (all files) 2024-11-12T10:53:57,882 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/B in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,882 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3fa6338f9288426b8211340be2f59e15, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c52ef74a433b4f6aae853109eca2d9d1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/6120fc31021f4805aae0d19824b04f14] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=36.8 K 2024-11-12T10:53:57,882 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:57,883 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/A is initiating minor compaction (all files) 2024-11-12T10:53:57,883 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/A in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,883 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/6ce5659b2f484ae5871bffc912dd57db, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/1b0a5e20d0194714af5327b0a7b64409, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/c1abb3b11325432c9be2457b7914d31e] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=41.6 K 2024-11-12T10:53:57,883 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fa6338f9288426b8211340be2f59e15, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731408834676 2024-11-12T10:53:57,883 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ce5659b2f484ae5871bffc912dd57db, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731408834676 2024-11-12T10:53:57,884 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting c52ef74a433b4f6aae853109eca2d9d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731408834703 2024-11-12T10:53:57,884 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b0a5e20d0194714af5327b0a7b64409, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731408834703 2024-11-12T10:53:57,884 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 6120fc31021f4805aae0d19824b04f14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731408835867 2024-11-12T10:53:57,885 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1abb3b11325432c9be2457b7914d31e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731408835867 2024-11-12T10:53:57,896 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#B#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:57,897 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/e9f20fc6caf7437696afe3aeb1ba2cf4 is 50, key is test_row_0/B:col10/1731408835868/Put/seqid=0 2024-11-12T10:53:57,904 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#A#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:57,905 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/5011b88e1d70451596d81c10561aa49a is 50, key is test_row_0/A:col10/1731408835868/Put/seqid=0 2024-11-12T10:53:57,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741918_1094 (size=13221) 2024-11-12T10:53:57,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741919_1095 (size=13221) 2024-11-12T10:53:57,932 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/5011b88e1d70451596d81c10561aa49a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/5011b88e1d70451596d81c10561aa49a 2024-11-12T10:53:57,939 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/A of b3cb1a6227d163d89478d6e9738e8055 into 5011b88e1d70451596d81c10561aa49a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:57,939 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:57,939 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/A, priority=13, startTime=1731408837880; duration=0sec 2024-11-12T10:53:57,939 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:57,939 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:A 2024-11-12T10:53:57,939 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:57,940 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:57,941 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/C is initiating minor compaction (all files) 2024-11-12T10:53:57,941 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/C in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,941 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6c207a0ca5fd44388c3c08012e4c7103, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/dc2e79b4c02b46ecb8aae2462213e26a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9ea63c9f00b2497fbbfd22a4bbb2d473] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=36.8 K 2024-11-12T10:53:57,941 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c207a0ca5fd44388c3c08012e4c7103, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731408834676 2024-11-12T10:53:57,942 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc2e79b4c02b46ecb8aae2462213e26a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731408834703 2024-11-12T10:53:57,942 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ea63c9f00b2497fbbfd22a4bbb2d473, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731408835867 2024-11-12T10:53:57,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T10:53:57,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:57,949 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T10:53:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:57,953 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#C#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:57,953 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/6efa522cf70143bc96d69c62b28121ad is 50, key is test_row_0/C:col10/1731408835868/Put/seqid=0 2024-11-12T10:53:57,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/d93aac77fb624807b861173544f78a48 is 50, key is test_row_0/A:col10/1731408836517/Put/seqid=0 2024-11-12T10:53:57,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741921_1097 (size=12301) 2024-11-12T10:53:57,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741920_1096 (size=13221) 2024-11-12T10:53:57,968 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/d93aac77fb624807b861173544f78a48 2024-11-12T10:53:57,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/429ea9ff00e34c6a8f28465cd2301b71 is 50, key is test_row_0/B:col10/1731408836517/Put/seqid=0 2024-11-12T10:53:57,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741922_1098 (size=12301) 2024-11-12T10:53:57,983 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/429ea9ff00e34c6a8f28465cd2301b71 2024-11-12T10:53:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/4171bfa4d406400d971a6b4fe75806a8 is 50, key is test_row_0/C:col10/1731408836517/Put/seqid=0 2024-11-12T10:53:58,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741923_1099 (size=12301) 2024-11-12T10:53:58,004 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/4171bfa4d406400d971a6b4fe75806a8 2024-11-12T10:53:58,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/d93aac77fb624807b861173544f78a48 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/d93aac77fb624807b861173544f78a48 2024-11-12T10:53:58,018 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/d93aac77fb624807b861173544f78a48, entries=150, sequenceid=428, filesize=12.0 K 2024-11-12T10:53:58,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/429ea9ff00e34c6a8f28465cd2301b71 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/429ea9ff00e34c6a8f28465cd2301b71 2024-11-12T10:53:58,029 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/429ea9ff00e34c6a8f28465cd2301b71, entries=150, sequenceid=428, filesize=12.0 K 2024-11-12T10:53:58,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/4171bfa4d406400d971a6b4fe75806a8 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/4171bfa4d406400d971a6b4fe75806a8 2024-11-12T10:53:58,045 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/4171bfa4d406400d971a6b4fe75806a8, entries=150, sequenceid=428, filesize=12.0 K 2024-11-12T10:53:58,048 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for b3cb1a6227d163d89478d6e9738e8055 in 99ms, sequenceid=428, compaction requested=false 2024-11-12T10:53:58,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:58,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:58,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-12T10:53:58,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-12T10:53:58,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-12T10:53:58,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6490 sec 2024-11-12T10:53:58,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6580 sec 2024-11-12T10:53:58,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:58,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T10:53:58,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:58,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:58,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:58,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:58,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:58,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:53:58,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/6fac98faab404e3c8df98b1c27073774 is 50, key is test_row_1/A:col10/1731408838066/Put/seqid=0 2024-11-12T10:53:58,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741924_1100 (size=17177) 2024-11-12T10:53:58,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408838123, max timestamp=1731408838136Num uniques 859; 2024-11-12T10:53:58,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:53:58,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408838123, max timestamp=1731408838136Num uniques 859; 2024-11-12T10:53:58,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:53:58,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408838123, max timestamp=1731408838136Num uniques 859; 2024-11-12T10:53:58,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:53:58,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:58,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408898237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:58,317 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/e9f20fc6caf7437696afe3aeb1ba2cf4 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/e9f20fc6caf7437696afe3aeb1ba2cf4 2024-11-12T10:53:58,322 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/B of b3cb1a6227d163d89478d6e9738e8055 into e9f20fc6caf7437696afe3aeb1ba2cf4(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:58,323 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:58,323 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/B, priority=13, startTime=1731408837881; duration=0sec 2024-11-12T10:53:58,323 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:58,323 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:B 2024-11-12T10:53:58,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408898347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:58,375 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/6efa522cf70143bc96d69c62b28121ad as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6efa522cf70143bc96d69c62b28121ad 2024-11-12T10:53:58,384 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/C of b3cb1a6227d163d89478d6e9738e8055 into 6efa522cf70143bc96d69c62b28121ad(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:58,384 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:58,384 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/C, priority=13, startTime=1731408837881; duration=0sec 2024-11-12T10:53:58,385 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:58,385 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:C 2024-11-12T10:53:58,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/6fac98faab404e3c8df98b1c27073774 2024-11-12T10:53:58,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/db5263029ff541ed965df3253cfa1109 is 50, key is test_row_1/B:col10/1731408838066/Put/seqid=0 2024-11-12T10:53:58,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741925_1101 (size=9857) 2024-11-12T10:53:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T10:53:58,547 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:53:58,547 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:53:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-12T10:53:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T10:53:58,550 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:53:58,550 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:53:58,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:53:58,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:58,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408898557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T10:53:58,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38590 deadline: 1731408898667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:58,670 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in b3cb1a6227d163d89478d6e9738e8055 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:53:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:58,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38544 deadline: 1731408898677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:58,682 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in b3cb1a6227d163d89478d6e9738e8055 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:53:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:58,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38534 deadline: 1731408898696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:58,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38560 deadline: 1731408898697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:58,700 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in b3cb1a6227d163d89478d6e9738e8055 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:53:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,700 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in b3cb1a6227d163d89478d6e9738e8055 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:53:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:58,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T10:53:58,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:58,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:58,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:58,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:58,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:58,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T10:53:58,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:58,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:58,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:58,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:58,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T10:53:58,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:53:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38520 deadline: 1731408898868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 2024-11-12T10:53:58,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:53:58,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:53:58,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/db5263029ff541ed965df3253cfa1109 2024-11-12T10:53:58,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/070b0368cd774f8686ff775b3b8964d9 is 50, key is test_row_1/C:col10/1731408838066/Put/seqid=0 2024-11-12T10:53:58,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741926_1102 (size=9857) 2024-11-12T10:53:58,980 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-12T10:53:58,981 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:53:58,981 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-12T10:53:58,981 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:53:58,982 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-12T10:53:58,982 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:53:58,985 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-12T10:53:58,985 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:53:58,986 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:58,986 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:58,986 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:58,986 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:58,986 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:58,986 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:53:58,986 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:53:58,986 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:53:58,986 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:58,986 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:58,987 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:58,987 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:53:59,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T10:53:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T10:53:59,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:59,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T10:53:59,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/070b0368cd774f8686ff775b3b8964d9 2024-11-12T10:53:59,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T10:53:59,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:59,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/6fac98faab404e3c8df98b1c27073774 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/6fac98faab404e3c8df98b1c27073774 2024-11-12T10:53:59,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/6fac98faab404e3c8df98b1c27073774, entries=250, sequenceid=438, filesize=16.8 K 2024-11-12T10:53:59,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/db5263029ff541ed965df3253cfa1109 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/db5263029ff541ed965df3253cfa1109 2024-11-12T10:53:59,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/db5263029ff541ed965df3253cfa1109, entries=100, sequenceid=438, filesize=9.6 K 2024-11-12T10:53:59,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/070b0368cd774f8686ff775b3b8964d9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/070b0368cd774f8686ff775b3b8964d9 2024-11-12T10:53:59,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/070b0368cd774f8686ff775b3b8964d9, entries=100, sequenceid=438, filesize=9.6 K 2024-11-12T10:53:59,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b3cb1a6227d163d89478d6e9738e8055 in 1277ms, sequenceid=438, compaction requested=true 2024-11-12T10:53:59,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:59,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:53:59,345 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:59,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:59,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:53:59,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:59,345 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:59,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cb1a6227d163d89478d6e9738e8055:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:53:59,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:59,346 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:59,346 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:59,346 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/A is initiating minor compaction (all files) 2024-11-12T10:53:59,346 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/B is initiating minor compaction (all files) 2024-11-12T10:53:59,346 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/B in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,346 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/A in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,346 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/e9f20fc6caf7437696afe3aeb1ba2cf4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/429ea9ff00e34c6a8f28465cd2301b71, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/db5263029ff541ed965df3253cfa1109] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=34.5 K 2024-11-12T10:53:59,346 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/5011b88e1d70451596d81c10561aa49a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/d93aac77fb624807b861173544f78a48, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/6fac98faab404e3c8df98b1c27073774] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=41.7 K 2024-11-12T10:53:59,346 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5011b88e1d70451596d81c10561aa49a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731408835867 2024-11-12T10:53:59,346 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting e9f20fc6caf7437696afe3aeb1ba2cf4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731408835867 2024-11-12T10:53:59,347 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting d93aac77fb624807b861173544f78a48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731408836517 2024-11-12T10:53:59,347 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 429ea9ff00e34c6a8f28465cd2301b71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731408836517 2024-11-12T10:53:59,347 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fac98faab404e3c8df98b1c27073774, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731408838054 2024-11-12T10:53:59,347 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting db5263029ff541ed965df3253cfa1109, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731408838062 2024-11-12T10:53:59,357 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#B#compaction#90 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:59,358 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/d419c803d66d4327a0c7e50a67ef5fa2 is 50, key is test_row_0/B:col10/1731408836517/Put/seqid=0 2024-11-12T10:53:59,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741927_1103 (size=13323) 2024-11-12T10:53:59,377 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#A#compaction#91 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:59,378 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/0fa71256cb3344f7b1831201d04f2b39 is 50, key is test_row_0/A:col10/1731408836517/Put/seqid=0 2024-11-12T10:53:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:53:59,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-12T10:53:59,379 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-12T10:53:59,379 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:53:59,379 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:59,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:53:59,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T10:53:59,379 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:53:59,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:53:59,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T10:53:59,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:53:59,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T10:53:59,380 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:53:59,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/efdc273b46014d409bf4cf87ee549208 is 50, key is test_row_0/A:col10/1731408838215/Put/seqid=0 2024-11-12T10:53:59,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741928_1104 (size=13323) 2024-11-12T10:53:59,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741929_1105 (size=12301) 2024-11-12T10:53:59,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T10:53:59,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:59,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T10:53:59,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:59,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T10:53:59,777 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/d419c803d66d4327a0c7e50a67ef5fa2 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/d419c803d66d4327a0c7e50a67ef5fa2 2024-11-12T10:53:59,783 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/B of b3cb1a6227d163d89478d6e9738e8055 into d419c803d66d4327a0c7e50a67ef5fa2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:59,783 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:59,783 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/B, priority=13, startTime=1731408839345; duration=0sec 2024-11-12T10:53:59,784 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:53:59,784 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:B 2024-11-12T10:53:59,784 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:53:59,785 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:53:59,785 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): b3cb1a6227d163d89478d6e9738e8055/C is initiating minor compaction (all files) 2024-11-12T10:53:59,785 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cb1a6227d163d89478d6e9738e8055/C in TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,785 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6efa522cf70143bc96d69c62b28121ad, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/4171bfa4d406400d971a6b4fe75806a8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/070b0368cd774f8686ff775b3b8964d9] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp, totalSize=34.5 K 2024-11-12T10:53:59,786 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 6efa522cf70143bc96d69c62b28121ad, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731408835867 2024-11-12T10:53:59,786 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 4171bfa4d406400d971a6b4fe75806a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731408836517 2024-11-12T10:53:59,787 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 070b0368cd774f8686ff775b3b8964d9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731408838062 2024-11-12T10:53:59,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T10:53:59,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:59,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,798 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/0fa71256cb3344f7b1831201d04f2b39 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/0fa71256cb3344f7b1831201d04f2b39 2024-11-12T10:53:59,798 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cb1a6227d163d89478d6e9738e8055#C#compaction#93 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:53:59,799 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/51468ddbe623494c98248e89b4813dd7 is 50, key is test_row_0/C:col10/1731408836517/Put/seqid=0 2024-11-12T10:53:59,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/efdc273b46014d409bf4cf87ee549208 2024-11-12T10:53:59,805 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/A of b3cb1a6227d163d89478d6e9738e8055 into 0fa71256cb3344f7b1831201d04f2b39(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:59,805 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:59,805 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/A, priority=13, startTime=1731408839345; duration=0sec 2024-11-12T10:53:59,805 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:59,805 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:A 2024-11-12T10:53:59,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741930_1106 (size=13323) 2024-11-12T10:53:59,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/8ab02150b7b94f83a5123512e04fc1ce is 50, key is test_row_0/B:col10/1731408838215/Put/seqid=0 2024-11-12T10:53:59,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741931_1107 (size=12301) 2024-11-12T10:53:59,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/8ab02150b7b94f83a5123512e04fc1ce 2024-11-12T10:53:59,815 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/51468ddbe623494c98248e89b4813dd7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/51468ddbe623494c98248e89b4813dd7 2024-11-12T10:53:59,821 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cb1a6227d163d89478d6e9738e8055/C of b3cb1a6227d163d89478d6e9738e8055 into 51468ddbe623494c98248e89b4813dd7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:53:59,821 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:53:59,821 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., storeName=b3cb1a6227d163d89478d6e9738e8055/C, priority=13, startTime=1731408839345; duration=0sec 2024-11-12T10:53:59,821 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:53:59,821 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cb1a6227d163d89478d6e9738e8055:C 2024-11-12T10:53:59,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/eb0b95c2c75c4590a37b0b6e71475d28 is 50, key is test_row_0/C:col10/1731408838215/Put/seqid=0 2024-11-12T10:53:59,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741932_1108 (size=12301) 2024-11-12T10:53:59,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T10:53:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:53:59,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:53:59,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:53:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:00,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T10:54:00,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:54:00,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:54:00,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:54:00,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:00,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:00,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/eb0b95c2c75c4590a37b0b6e71475d28 2024-11-12T10:54:00,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/efdc273b46014d409bf4cf87ee549208 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/efdc273b46014d409bf4cf87ee549208 2024-11-12T10:54:00,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/efdc273b46014d409bf4cf87ee549208, entries=150, sequenceid=478, filesize=12.0 K 2024-11-12T10:54:00,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/8ab02150b7b94f83a5123512e04fc1ce as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/8ab02150b7b94f83a5123512e04fc1ce 2024-11-12T10:54:00,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/8ab02150b7b94f83a5123512e04fc1ce, entries=150, sequenceid=478, filesize=12.0 K 2024-11-12T10:54:00,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/eb0b95c2c75c4590a37b0b6e71475d28 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/eb0b95c2c75c4590a37b0b6e71475d28 2024-11-12T10:54:00,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T10:54:00,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:54:00,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. as already flushing 2024-11-12T10:54:00,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:54:00,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:00,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:00,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/eb0b95c2c75c4590a37b0b6e71475d28, entries=150, sequenceid=478, filesize=12.0 K 2024-11-12T10:54:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:00,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=0 B/0 for b3cb1a6227d163d89478d6e9738e8055 in 880ms, sequenceid=478, compaction requested=false 2024-11-12T10:54:00,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:54:00,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T10:54:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:54:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for b3cb1a6227d163d89478d6e9738e8055: 2024-11-12T10:54:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:54:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-12T10:54:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-12T10:54:00,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-12T10:54:00,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8660 sec 2024-11-12T10:54:00,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.8730 sec 2024-11-12T10:54:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T10:54:00,699 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:00,702 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-12T10:54:00,702 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-12T10:54:00,702 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:00,702 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:00,702 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:00,702 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:00,702 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:00,702 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:00,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:00,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:00,719 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-12T10:54:00,719 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-12T10:54:00,719 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:00,719 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:00,719 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:00,719 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:00,719 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:00,719 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:00,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:00,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:00,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T10:54:00,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-12T10:54:00,720 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-12T10:54:00,720 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-12T10:54:00,720 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-12T10:54:00,720 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-12T10:54:00,720 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T10:54:00,720 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2753 2024-11-12T10:54:00,720 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2821 2024-11-12T10:54:00,720 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T10:54:00,720 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1228 2024-11-12T10:54:00,720 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3680 rows 2024-11-12T10:54:00,720 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1222 2024-11-12T10:54:00,720 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3662 rows 2024-11-12T10:54:00,720 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T10:54:00,720 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T10:54:00,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:00,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:00,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:00,725 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T10:54:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T10:54:00,735 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408840734"}]},"ts":"1731408840734"} 2024-11-12T10:54:00,736 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T10:54:00,736 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T10:54:00,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T10:54:00,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cb1a6227d163d89478d6e9738e8055, UNASSIGN}] 2024-11-12T10:54:00,743 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cb1a6227d163d89478d6e9738e8055, UNASSIGN 2024-11-12T10:54:00,744 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b3cb1a6227d163d89478d6e9738e8055, regionState=CLOSING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:54:00,747 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cb1a6227d163d89478d6e9738e8055, UNASSIGN because future has completed 2024-11-12T10:54:00,747 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T10:54:00,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:54:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T10:54:00,905 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:54:00,905 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T10:54:00,906 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing b3cb1a6227d163d89478d6e9738e8055, disabling compactions & flushes 2024-11-12T10:54:00,907 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:54:00,907 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:54:00,907 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. after waiting 0 ms 2024-11-12T10:54:00,907 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:54:00,907 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing b3cb1a6227d163d89478d6e9738e8055 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T10:54:00,908 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=A 2024-11-12T10:54:00,908 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:00,908 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=B 2024-11-12T10:54:00,908 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:00,908 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3cb1a6227d163d89478d6e9738e8055, store=C 2024-11-12T10:54:00,908 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:00,916 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/385f56a150514499a582c5fb09f98c6f is 50, key is test_row_0/A:col10/1731408840717/Put/seqid=0 2024-11-12T10:54:00,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741933_1109 (size=12301) 2024-11-12T10:54:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T10:54:01,322 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/385f56a150514499a582c5fb09f98c6f 2024-11-12T10:54:01,338 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/31a3406a68fd407a9d367a8ac4a847c8 is 50, key is test_row_0/B:col10/1731408840717/Put/seqid=0 2024-11-12T10:54:01,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741934_1110 (size=12301) 2024-11-12T10:54:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T10:54:01,377 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T10:54:01,744 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/31a3406a68fd407a9d367a8ac4a847c8 2024-11-12T10:54:01,761 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/8d2e26052c5a4b9395dfc12c1f0c8192 is 50, key is test_row_0/C:col10/1731408840717/Put/seqid=0 2024-11-12T10:54:01,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741935_1111 (size=12301) 2024-11-12T10:54:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T10:54:02,166 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/8d2e26052c5a4b9395dfc12c1f0c8192 2024-11-12T10:54:02,173 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/A/385f56a150514499a582c5fb09f98c6f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/385f56a150514499a582c5fb09f98c6f 2024-11-12T10:54:02,180 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/385f56a150514499a582c5fb09f98c6f, entries=150, sequenceid=489, filesize=12.0 K 2024-11-12T10:54:02,181 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/B/31a3406a68fd407a9d367a8ac4a847c8 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/31a3406a68fd407a9d367a8ac4a847c8 2024-11-12T10:54:02,185 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/31a3406a68fd407a9d367a8ac4a847c8, entries=150, sequenceid=489, filesize=12.0 K 2024-11-12T10:54:02,186 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/.tmp/C/8d2e26052c5a4b9395dfc12c1f0c8192 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8d2e26052c5a4b9395dfc12c1f0c8192 2024-11-12T10:54:02,190 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8d2e26052c5a4b9395dfc12c1f0c8192, entries=150, sequenceid=489, filesize=12.0 K 2024-11-12T10:54:02,191 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b3cb1a6227d163d89478d6e9738e8055 in 1284ms, sequenceid=489, compaction requested=true 2024-11-12T10:54:02,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/c46d8b0051db40e8998fc7f7f0197c00, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ee80f3ebab8449b9b7b5082dfdbe9855, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/e6e4b24294e640c784d637c912e23f5f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/74447cab94084619b9f42b3674cb1480, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/d92ce7472b8c41639f87132beeb870c5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ae41ff35e7194a42ad0ff9ca760e5423, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/91cadca5c25d47339f787f2ebe2acd99, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/e0fb839f604049f49abbc6d7627f3a0c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/041781a22c354d8a989cf8d9ac4ace23, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/96b096c33d514da8b46aa6ccdb259094, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ade19b01cc9a4e42bb428d0ff3625619, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/be32388cded648148aef55b0cd01ec6f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/b7d545ae206d4f02ab4aeae8cb7c5b53, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/eab8a08bc3bf4357aca09800c72a03c3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/fc433261a23a4afaa8f2b2af149b4244, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/44bb807c754e467c942ce93705c7b7de, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/b57cc4398e6444bc8d19652182bf0fbb, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/a0603fbf81bb41469c6d94e20fc77a3b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/78c24e93773c43378b505ddfa7a533ba, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/f7bd626ea4db44249341389dfafc953b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/5f948eccd8c94b50b921ab9b36a2bceb, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/23864e6dc08847b282e1aa5a382f0a56, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/432a8791d6b14b8da93f75edeeb69077, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/4de9a153e39442d680274806a261a5dc, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/6ce5659b2f484ae5871bffc912dd57db, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/1b0a5e20d0194714af5327b0a7b64409, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/5011b88e1d70451596d81c10561aa49a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/c1abb3b11325432c9be2457b7914d31e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/d93aac77fb624807b861173544f78a48, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/6fac98faab404e3c8df98b1c27073774] to archive 2024-11-12T10:54:02,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:54:02,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/c46d8b0051db40e8998fc7f7f0197c00 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/c46d8b0051db40e8998fc7f7f0197c00 2024-11-12T10:54:02,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ee80f3ebab8449b9b7b5082dfdbe9855 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ee80f3ebab8449b9b7b5082dfdbe9855 2024-11-12T10:54:02,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/e6e4b24294e640c784d637c912e23f5f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/e6e4b24294e640c784d637c912e23f5f 2024-11-12T10:54:02,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/74447cab94084619b9f42b3674cb1480 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/74447cab94084619b9f42b3674cb1480 2024-11-12T10:54:02,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/d92ce7472b8c41639f87132beeb870c5 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/d92ce7472b8c41639f87132beeb870c5 2024-11-12T10:54:02,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ae41ff35e7194a42ad0ff9ca760e5423 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ae41ff35e7194a42ad0ff9ca760e5423 2024-11-12T10:54:02,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/91cadca5c25d47339f787f2ebe2acd99 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/91cadca5c25d47339f787f2ebe2acd99 2024-11-12T10:54:02,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/e0fb839f604049f49abbc6d7627f3a0c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/e0fb839f604049f49abbc6d7627f3a0c 2024-11-12T10:54:02,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/041781a22c354d8a989cf8d9ac4ace23 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/041781a22c354d8a989cf8d9ac4ace23 2024-11-12T10:54:02,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/96b096c33d514da8b46aa6ccdb259094 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/96b096c33d514da8b46aa6ccdb259094 2024-11-12T10:54:02,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ade19b01cc9a4e42bb428d0ff3625619 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/ade19b01cc9a4e42bb428d0ff3625619 2024-11-12T10:54:02,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/be32388cded648148aef55b0cd01ec6f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/be32388cded648148aef55b0cd01ec6f 2024-11-12T10:54:02,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/b7d545ae206d4f02ab4aeae8cb7c5b53 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/b7d545ae206d4f02ab4aeae8cb7c5b53 2024-11-12T10:54:02,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/eab8a08bc3bf4357aca09800c72a03c3 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/eab8a08bc3bf4357aca09800c72a03c3 2024-11-12T10:54:02,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/fc433261a23a4afaa8f2b2af149b4244 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/fc433261a23a4afaa8f2b2af149b4244 2024-11-12T10:54:02,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/44bb807c754e467c942ce93705c7b7de to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/44bb807c754e467c942ce93705c7b7de 2024-11-12T10:54:02,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/b57cc4398e6444bc8d19652182bf0fbb to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/b57cc4398e6444bc8d19652182bf0fbb 2024-11-12T10:54:02,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/a0603fbf81bb41469c6d94e20fc77a3b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/a0603fbf81bb41469c6d94e20fc77a3b 2024-11-12T10:54:02,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/78c24e93773c43378b505ddfa7a533ba to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/78c24e93773c43378b505ddfa7a533ba 2024-11-12T10:54:02,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/f7bd626ea4db44249341389dfafc953b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/f7bd626ea4db44249341389dfafc953b 2024-11-12T10:54:02,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/5f948eccd8c94b50b921ab9b36a2bceb to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/5f948eccd8c94b50b921ab9b36a2bceb 2024-11-12T10:54:02,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/23864e6dc08847b282e1aa5a382f0a56 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/23864e6dc08847b282e1aa5a382f0a56 2024-11-12T10:54:02,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/432a8791d6b14b8da93f75edeeb69077 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/432a8791d6b14b8da93f75edeeb69077 2024-11-12T10:54:02,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/4de9a153e39442d680274806a261a5dc to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/4de9a153e39442d680274806a261a5dc 2024-11-12T10:54:02,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/6ce5659b2f484ae5871bffc912dd57db to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/6ce5659b2f484ae5871bffc912dd57db 2024-11-12T10:54:02,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/1b0a5e20d0194714af5327b0a7b64409 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/1b0a5e20d0194714af5327b0a7b64409 2024-11-12T10:54:02,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/5011b88e1d70451596d81c10561aa49a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/5011b88e1d70451596d81c10561aa49a 2024-11-12T10:54:02,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/c1abb3b11325432c9be2457b7914d31e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/c1abb3b11325432c9be2457b7914d31e 2024-11-12T10:54:02,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/d93aac77fb624807b861173544f78a48 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/d93aac77fb624807b861173544f78a48 2024-11-12T10:54:02,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/6fac98faab404e3c8df98b1c27073774 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/6fac98faab404e3c8df98b1c27073774 2024-11-12T10:54:02,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c90362070ef1422786371c514a630206, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/a6112230b1154887867f5ba680f9159e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/73ca166779584788ae5f2f23c72bfda9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/d4a6dafe7a1847db96a43b8ecf192b4c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f3f9424f508548709d7ea185967be478, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/61c9d781ae774bc3bccf98f736498698, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/8ed1a1443b03415b9c46b8012d19ae8b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/ade96c19347b4171aaa878dd3f62efe4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c70c6a742cb542a38060d9d1763b1d47, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c916786713f148f4a4e6ee4b2a4c0f6f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/1375775e70f241f2bac41548eb50a8dc, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f39d49880c5946f1a325e1f9f7c40f00, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f4183ad64a5c44cc918a7ad3e6a6ec75, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3221939eb2fa439da30b5868f550ece1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/6116f37387034dfda0c5831e3d36a95a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/accf259587d841dc8352570536488e29, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3cf2c477fc9740d4a5b48b2064a4fd64, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/24749ccad05344e899ff0958b10173eb, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/fcc1b73f8b0047a9aa48b91d36fb3a86, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/68c96eadc8fa4f0f84479082b909b60d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/57feb8d3ce4346bbb9eaa63e9d544dce, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/4d652dd3fdba48a995230f01381d0953, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/2f4048ce3af143db942969b7875028e1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3fa6338f9288426b8211340be2f59e15, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3642bfe6b04b41de96921c6e493dffb3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c52ef74a433b4f6aae853109eca2d9d1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/e9f20fc6caf7437696afe3aeb1ba2cf4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/6120fc31021f4805aae0d19824b04f14, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/429ea9ff00e34c6a8f28465cd2301b71, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/db5263029ff541ed965df3253cfa1109] to archive 2024-11-12T10:54:02,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:54:02,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c90362070ef1422786371c514a630206 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c90362070ef1422786371c514a630206 2024-11-12T10:54:02,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/a6112230b1154887867f5ba680f9159e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/a6112230b1154887867f5ba680f9159e 2024-11-12T10:54:02,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/73ca166779584788ae5f2f23c72bfda9 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/73ca166779584788ae5f2f23c72bfda9 2024-11-12T10:54:02,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/d4a6dafe7a1847db96a43b8ecf192b4c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/d4a6dafe7a1847db96a43b8ecf192b4c 2024-11-12T10:54:02,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f3f9424f508548709d7ea185967be478 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f3f9424f508548709d7ea185967be478 2024-11-12T10:54:02,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/61c9d781ae774bc3bccf98f736498698 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/61c9d781ae774bc3bccf98f736498698 2024-11-12T10:54:02,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/8ed1a1443b03415b9c46b8012d19ae8b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/8ed1a1443b03415b9c46b8012d19ae8b 2024-11-12T10:54:02,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/ade96c19347b4171aaa878dd3f62efe4 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/ade96c19347b4171aaa878dd3f62efe4 2024-11-12T10:54:02,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c70c6a742cb542a38060d9d1763b1d47 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c70c6a742cb542a38060d9d1763b1d47 2024-11-12T10:54:02,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c916786713f148f4a4e6ee4b2a4c0f6f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c916786713f148f4a4e6ee4b2a4c0f6f 2024-11-12T10:54:02,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/1375775e70f241f2bac41548eb50a8dc to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/1375775e70f241f2bac41548eb50a8dc 2024-11-12T10:54:02,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f39d49880c5946f1a325e1f9f7c40f00 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f39d49880c5946f1a325e1f9f7c40f00 2024-11-12T10:54:02,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f4183ad64a5c44cc918a7ad3e6a6ec75 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/f4183ad64a5c44cc918a7ad3e6a6ec75 2024-11-12T10:54:02,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3221939eb2fa439da30b5868f550ece1 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3221939eb2fa439da30b5868f550ece1 2024-11-12T10:54:02,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/6116f37387034dfda0c5831e3d36a95a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/6116f37387034dfda0c5831e3d36a95a 2024-11-12T10:54:02,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/accf259587d841dc8352570536488e29 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/accf259587d841dc8352570536488e29 2024-11-12T10:54:02,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3cf2c477fc9740d4a5b48b2064a4fd64 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3cf2c477fc9740d4a5b48b2064a4fd64 2024-11-12T10:54:02,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/24749ccad05344e899ff0958b10173eb to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/24749ccad05344e899ff0958b10173eb 2024-11-12T10:54:02,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/fcc1b73f8b0047a9aa48b91d36fb3a86 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/fcc1b73f8b0047a9aa48b91d36fb3a86 2024-11-12T10:54:02,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/68c96eadc8fa4f0f84479082b909b60d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/68c96eadc8fa4f0f84479082b909b60d 2024-11-12T10:54:02,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/57feb8d3ce4346bbb9eaa63e9d544dce to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/57feb8d3ce4346bbb9eaa63e9d544dce 2024-11-12T10:54:02,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/4d652dd3fdba48a995230f01381d0953 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/4d652dd3fdba48a995230f01381d0953 2024-11-12T10:54:02,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/2f4048ce3af143db942969b7875028e1 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/2f4048ce3af143db942969b7875028e1 2024-11-12T10:54:02,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3fa6338f9288426b8211340be2f59e15 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3fa6338f9288426b8211340be2f59e15 2024-11-12T10:54:02,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3642bfe6b04b41de96921c6e493dffb3 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/3642bfe6b04b41de96921c6e493dffb3 2024-11-12T10:54:02,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c52ef74a433b4f6aae853109eca2d9d1 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/c52ef74a433b4f6aae853109eca2d9d1 2024-11-12T10:54:02,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/e9f20fc6caf7437696afe3aeb1ba2cf4 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/e9f20fc6caf7437696afe3aeb1ba2cf4 2024-11-12T10:54:02,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/6120fc31021f4805aae0d19824b04f14 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/6120fc31021f4805aae0d19824b04f14 2024-11-12T10:54:02,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/429ea9ff00e34c6a8f28465cd2301b71 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/429ea9ff00e34c6a8f28465cd2301b71 2024-11-12T10:54:02,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/db5263029ff541ed965df3253cfa1109 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/db5263029ff541ed965df3253cfa1109 2024-11-12T10:54:02,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/ef43ae91ec7448bf921a525fe97157f8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/26ed2a36f5a14da7a915146c84498228, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/2e924ed5445f4fd999e1853ffcd672fc, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/d360388004a44fd9a38bdb88e31d528c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/bb8e6d29b4b743d79a50763dcacafc7c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/1a9f642212244c7da17a3f174db6a9b2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/0f54bcbb4ff8408d97dea2cbb15bbd33, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/7fdc6b3a4f17472796efd2adad86998b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/232dcfffeb40479189cfc46f64073569, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9d4abd08ec614fc1a33e4d7861cc8ffb, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/f06d6b8cd52d42f8bfea900c00bfb913, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/5efd170583c64f9b9888704a714b6166, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6bafaee6953944a3b93d0e2942aae6f7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9a7b1c42adf047b5923864026e73dc23, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6eb998eb6b544021b36aa784d8a20231, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/318721ca80cd40e2b4d2255513ec9df9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/cd0a414aff714638aeaa9411ba754c90, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/2f23e8b890c74735b8d6ee67b2d0dac5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8dac532863af40f5bb3e79154da0031c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/77e8b075ab2c44eabe4528f2b37c0f83, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/e0e531500a68413f8ac691130ec7049a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/ad129db58b564a0e92f02a5ddceffb9c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8ff00deabe644c678ade523d1d320493, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6c207a0ca5fd44388c3c08012e4c7103, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/05cb465732c34a7d9ae40125bbba58f4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/dc2e79b4c02b46ecb8aae2462213e26a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6efa522cf70143bc96d69c62b28121ad, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9ea63c9f00b2497fbbfd22a4bbb2d473, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/4171bfa4d406400d971a6b4fe75806a8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/070b0368cd774f8686ff775b3b8964d9] to archive 2024-11-12T10:54:02,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:54:02,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/ef43ae91ec7448bf921a525fe97157f8 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/ef43ae91ec7448bf921a525fe97157f8 2024-11-12T10:54:02,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/26ed2a36f5a14da7a915146c84498228 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/26ed2a36f5a14da7a915146c84498228 2024-11-12T10:54:02,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/2e924ed5445f4fd999e1853ffcd672fc to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/2e924ed5445f4fd999e1853ffcd672fc 2024-11-12T10:54:02,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/d360388004a44fd9a38bdb88e31d528c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/d360388004a44fd9a38bdb88e31d528c 2024-11-12T10:54:02,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/bb8e6d29b4b743d79a50763dcacafc7c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/bb8e6d29b4b743d79a50763dcacafc7c 2024-11-12T10:54:02,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/1a9f642212244c7da17a3f174db6a9b2 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/1a9f642212244c7da17a3f174db6a9b2 2024-11-12T10:54:02,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/0f54bcbb4ff8408d97dea2cbb15bbd33 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/0f54bcbb4ff8408d97dea2cbb15bbd33 2024-11-12T10:54:02,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/7fdc6b3a4f17472796efd2adad86998b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/7fdc6b3a4f17472796efd2adad86998b 2024-11-12T10:54:02,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/232dcfffeb40479189cfc46f64073569 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/232dcfffeb40479189cfc46f64073569 2024-11-12T10:54:02,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9d4abd08ec614fc1a33e4d7861cc8ffb to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9d4abd08ec614fc1a33e4d7861cc8ffb 2024-11-12T10:54:02,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/f06d6b8cd52d42f8bfea900c00bfb913 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/f06d6b8cd52d42f8bfea900c00bfb913 2024-11-12T10:54:02,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/5efd170583c64f9b9888704a714b6166 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/5efd170583c64f9b9888704a714b6166 2024-11-12T10:54:02,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6bafaee6953944a3b93d0e2942aae6f7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6bafaee6953944a3b93d0e2942aae6f7 2024-11-12T10:54:02,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9a7b1c42adf047b5923864026e73dc23 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9a7b1c42adf047b5923864026e73dc23 2024-11-12T10:54:02,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6eb998eb6b544021b36aa784d8a20231 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6eb998eb6b544021b36aa784d8a20231 2024-11-12T10:54:02,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/318721ca80cd40e2b4d2255513ec9df9 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/318721ca80cd40e2b4d2255513ec9df9 2024-11-12T10:54:02,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/cd0a414aff714638aeaa9411ba754c90 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/cd0a414aff714638aeaa9411ba754c90 2024-11-12T10:54:02,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/2f23e8b890c74735b8d6ee67b2d0dac5 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/2f23e8b890c74735b8d6ee67b2d0dac5 2024-11-12T10:54:02,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8dac532863af40f5bb3e79154da0031c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8dac532863af40f5bb3e79154da0031c 2024-11-12T10:54:02,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/77e8b075ab2c44eabe4528f2b37c0f83 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/77e8b075ab2c44eabe4528f2b37c0f83 2024-11-12T10:54:02,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/e0e531500a68413f8ac691130ec7049a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/e0e531500a68413f8ac691130ec7049a 2024-11-12T10:54:02,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/ad129db58b564a0e92f02a5ddceffb9c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/ad129db58b564a0e92f02a5ddceffb9c 2024-11-12T10:54:02,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8ff00deabe644c678ade523d1d320493 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8ff00deabe644c678ade523d1d320493 2024-11-12T10:54:02,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6c207a0ca5fd44388c3c08012e4c7103 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6c207a0ca5fd44388c3c08012e4c7103 2024-11-12T10:54:02,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/05cb465732c34a7d9ae40125bbba58f4 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/05cb465732c34a7d9ae40125bbba58f4 2024-11-12T10:54:02,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/dc2e79b4c02b46ecb8aae2462213e26a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/dc2e79b4c02b46ecb8aae2462213e26a 2024-11-12T10:54:02,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6efa522cf70143bc96d69c62b28121ad to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/6efa522cf70143bc96d69c62b28121ad 2024-11-12T10:54:02,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9ea63c9f00b2497fbbfd22a4bbb2d473 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/9ea63c9f00b2497fbbfd22a4bbb2d473 2024-11-12T10:54:02,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/4171bfa4d406400d971a6b4fe75806a8 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/4171bfa4d406400d971a6b4fe75806a8 2024-11-12T10:54:02,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/070b0368cd774f8686ff775b3b8964d9 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/070b0368cd774f8686ff775b3b8964d9 2024-11-12T10:54:02,328 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/recovered.edits/492.seqid, newMaxSeqId=492, maxSeqId=1 2024-11-12T10:54:02,329 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. 2024-11-12T10:54:02,329 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for b3cb1a6227d163d89478d6e9738e8055: Waiting for close lock at 1731408840906Running coprocessor pre-close hooks at 1731408840906Disabling compacts and flushes for region at 1731408840906Disabling writes for close at 1731408840907 (+1 ms)Obtaining lock to block concurrent updates at 1731408840907Preparing flush snapshotting stores in b3cb1a6227d163d89478d6e9738e8055 at 1731408840907Finished memstore snapshotting TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731408840908 (+1 ms)Flushing stores of TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055. at 1731408840909 (+1 ms)Flushing b3cb1a6227d163d89478d6e9738e8055/A: creating writer at 1731408840909Flushing b3cb1a6227d163d89478d6e9738e8055/A: appending metadata at 1731408840916 (+7 ms)Flushing b3cb1a6227d163d89478d6e9738e8055/A: closing flushed file at 1731408840916Flushing b3cb1a6227d163d89478d6e9738e8055/B: creating writer at 1731408841334 (+418 ms)Flushing b3cb1a6227d163d89478d6e9738e8055/B: appending metadata at 1731408841338 (+4 ms)Flushing b3cb1a6227d163d89478d6e9738e8055/B: closing flushed file at 1731408841338Flushing b3cb1a6227d163d89478d6e9738e8055/C: creating writer at 1731408841756 (+418 ms)Flushing b3cb1a6227d163d89478d6e9738e8055/C: appending metadata at 1731408841760 (+4 ms)Flushing b3cb1a6227d163d89478d6e9738e8055/C: closing flushed file at 1731408841761 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a2f1257: reopening flushed file at 1731408842172 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@eace662: reopening flushed file at 1731408842180 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d1f45b1: reopening flushed file at 1731408842185 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b3cb1a6227d163d89478d6e9738e8055 in 1284ms, sequenceid=489, compaction requested=true at 1731408842191 (+6 ms)Writing region close event to WAL at 1731408842324 (+133 ms)Running coprocessor post-close hooks at 1731408842329 (+5 ms)Closed at 1731408842329 2024-11-12T10:54:02,332 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:54:02,332 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b3cb1a6227d163d89478d6e9738e8055, regionState=CLOSED 2024-11-12T10:54:02,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:54:02,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-12T10:54:02,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure b3cb1a6227d163d89478d6e9738e8055, server=aed202db7571,43143,1731408812834 in 1.5890 sec 2024-11-12T10:54:02,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-12T10:54:02,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cb1a6227d163d89478d6e9738e8055, UNASSIGN in 1.5960 sec 2024-11-12T10:54:02,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-12T10:54:02,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6020 sec 2024-11-12T10:54:02,343 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408842343"}]},"ts":"1731408842343"} 2024-11-12T10:54:02,345 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T10:54:02,345 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T10:54:02,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6180 sec 2024-11-12T10:54:02,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T10:54:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T10:54:02,878 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:02,881 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T10:54:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:02,886 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T10:54:02,888 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:02,894 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:54:02,898 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/recovered.edits] 2024-11-12T10:54:02,901 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/0fa71256cb3344f7b1831201d04f2b39 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/0fa71256cb3344f7b1831201d04f2b39 2024-11-12T10:54:02,903 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/385f56a150514499a582c5fb09f98c6f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/385f56a150514499a582c5fb09f98c6f 2024-11-12T10:54:02,904 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/efdc273b46014d409bf4cf87ee549208 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/A/efdc273b46014d409bf4cf87ee549208 2024-11-12T10:54:02,907 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/31a3406a68fd407a9d367a8ac4a847c8 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/31a3406a68fd407a9d367a8ac4a847c8 2024-11-12T10:54:02,908 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/8ab02150b7b94f83a5123512e04fc1ce to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/8ab02150b7b94f83a5123512e04fc1ce 2024-11-12T10:54:02,910 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/d419c803d66d4327a0c7e50a67ef5fa2 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/B/d419c803d66d4327a0c7e50a67ef5fa2 2024-11-12T10:54:02,912 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/51468ddbe623494c98248e89b4813dd7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/51468ddbe623494c98248e89b4813dd7 2024-11-12T10:54:02,914 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8d2e26052c5a4b9395dfc12c1f0c8192 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/8d2e26052c5a4b9395dfc12c1f0c8192 2024-11-12T10:54:02,915 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/eb0b95c2c75c4590a37b0b6e71475d28 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/C/eb0b95c2c75c4590a37b0b6e71475d28 2024-11-12T10:54:02,918 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/recovered.edits/492.seqid to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055/recovered.edits/492.seqid 2024-11-12T10:54:02,919 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/b3cb1a6227d163d89478d6e9738e8055 2024-11-12T10:54:02,919 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T10:54:02,925 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:02,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-12T10:54:02,936 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T10:54:02,970 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T10:54:02,972 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:02,972 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T10:54:02,972 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731408842972"}]},"ts":"9223372036854775807"} 2024-11-12T10:54:02,976 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T10:54:02,976 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b3cb1a6227d163d89478d6e9738e8055, NAME => 'TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T10:54:02,977 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T10:54:02,977 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731408842977"}]},"ts":"9223372036854775807"} 2024-11-12T10:54:02,980 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T10:54:02,981 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:02,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 99 msec 2024-11-12T10:54:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T10:54:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T10:54:02,999 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:03,011 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=250 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/aed202db7571:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1240038058_22 at /127.0.0.1:38598 [Waiting for operation #63] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2109716649_22 at /127.0.0.1:38600 [Waiting for operation #60] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2109716649_22 at /127.0.0.1:52944 [Waiting for operation #255] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1240038058_22 at /127.0.0.1:37906 [Waiting for operation #239] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS:0;aed202db7571:43143-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/aed202db7571:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=680 (was 737), ProcessCount=11 (was 11), AvailableMemoryMB=2217 (was 2779) 2024-11-12T10:54:03,019 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=680, ProcessCount=11, AvailableMemoryMB=2216 2024-11-12T10:54:03,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T10:54:03,021 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T10:54:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:03,023 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T10:54:03,023 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:03,024 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-12T10:54:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T10:54:03,025 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T10:54:03,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741936_1112 (size=1061) 2024-11-12T10:54:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T10:54:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T10:54:03,437 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98 2024-11-12T10:54:03,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741937_1113 (size=53) 2024-11-12T10:54:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T10:54:03,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:54:03,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f3fbab0ede17db49ead8375d789c7d6a, disabling compactions & flushes 2024-11-12T10:54:03,849 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:03,850 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:03,850 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. after waiting 0 ms 2024-11-12T10:54:03,850 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:03,850 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:03,850 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f3fbab0ede17db49ead8375d789c7d6a: Waiting for close lock at 1731408843849Disabling compacts and flushes for region at 1731408843849Disabling writes for close at 1731408843850 (+1 ms)Writing region close event to WAL at 1731408843850Closed at 1731408843850 2024-11-12T10:54:03,854 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T10:54:03,855 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731408843854"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731408843854"}]},"ts":"1731408843854"} 2024-11-12T10:54:03,859 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T10:54:03,860 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T10:54:03,861 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408843860"}]},"ts":"1731408843860"} 2024-11-12T10:54:03,862 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T10:54:03,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, ASSIGN}] 2024-11-12T10:54:03,864 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, ASSIGN 2024-11-12T10:54:03,865 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, ASSIGN; state=OFFLINE, location=aed202db7571,43143,1731408812834; forceNewPlan=false, retain=false 2024-11-12T10:54:04,015 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f3fbab0ede17db49ead8375d789c7d6a, regionState=OPENING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:54:04,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, ASSIGN because future has completed 2024-11-12T10:54:04,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:54:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T10:54:04,179 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:04,179 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:54:04,180 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:04,180 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:54:04,180 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:04,180 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:04,182 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:04,183 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:54:04,184 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fbab0ede17db49ead8375d789c7d6a columnFamilyName A 2024-11-12T10:54:04,185 DEBUG [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:04,185 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.HStore(327): Store=f3fbab0ede17db49ead8375d789c7d6a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:54:04,185 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:04,187 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:54:04,187 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fbab0ede17db49ead8375d789c7d6a columnFamilyName B 2024-11-12T10:54:04,187 DEBUG [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:04,187 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.HStore(327): Store=f3fbab0ede17db49ead8375d789c7d6a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:54:04,188 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:04,189 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:54:04,189 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fbab0ede17db49ead8375d789c7d6a columnFamilyName C 2024-11-12T10:54:04,189 DEBUG [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:04,190 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.HStore(327): Store=f3fbab0ede17db49ead8375d789c7d6a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:54:04,190 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:04,190 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:04,191 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:04,191 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:04,193 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:04,193 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:04,193 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T10:54:04,195 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:04,198 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:54:04,198 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened f3fbab0ede17db49ead8375d789c7d6a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59467446, jitterRate=-0.11386600136756897}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T10:54:04,198 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:04,200 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for f3fbab0ede17db49ead8375d789c7d6a: Running coprocessor pre-open hook at 1731408844180Writing region info on filesystem at 1731408844180Initializing all the Stores at 1731408844181 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408844181Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408844182 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408844182Cleaning up temporary data from old regions at 1731408844193 (+11 ms)Running coprocessor post-open hooks at 1731408844198 (+5 ms)Region opened successfully at 1731408844199 (+1 ms) 2024-11-12T10:54:04,201 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., pid=30, masterSystemTime=1731408844170 2024-11-12T10:54:04,203 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:04,204 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:04,204 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f3fbab0ede17db49ead8375d789c7d6a, regionState=OPEN, openSeqNum=2, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:54:04,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:54:04,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-12T10:54:04,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 in 189 msec 2024-11-12T10:54:04,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-12T10:54:04,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, ASSIGN in 346 msec 2024-11-12T10:54:04,213 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T10:54:04,213 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408844213"}]},"ts":"1731408844213"} 2024-11-12T10:54:04,215 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T10:54:04,216 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T10:54:04,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1950 sec 2024-11-12T10:54:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T10:54:05,179 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:05,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:05,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:05,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:05,187 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1896e41f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:05,190 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:05,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bdc2aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:05,191 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:05,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:05,195 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T10:54:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is aed202db7571,45901,1731408812240 2024-11-12T10:54:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63a47a4c 2024-11-12T10:54:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T10:54:05,197 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T10:54:05,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T10:54:05,203 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T10:54:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:05,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741938_1114 (size=1097) 2024-11-12T10:54:05,625 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T10:54:05,626 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T10:54:05,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T10:54:05,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, REOPEN/MOVE}] 2024-11-12T10:54:05,642 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, REOPEN/MOVE 2024-11-12T10:54:05,643 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f3fbab0ede17db49ead8375d789c7d6a, regionState=CLOSING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:54:05,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, REOPEN/MOVE because future has completed 2024-11-12T10:54:05,645 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T10:54:05,645 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:54:05,798 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:05,798 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T10:54:05,799 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing f3fbab0ede17db49ead8375d789c7d6a, disabling compactions & flushes 2024-11-12T10:54:05,799 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:05,799 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:05,799 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. after waiting 0 ms 2024-11-12T10:54:05,799 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:05,808 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T10:54:05,809 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:05,809 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for f3fbab0ede17db49ead8375d789c7d6a: Waiting for close lock at 1731408845799Running coprocessor pre-close hooks at 1731408845799Disabling compacts and flushes for region at 1731408845799Disabling writes for close at 1731408845799Writing region close event to WAL at 1731408845800 (+1 ms)Running coprocessor post-close hooks at 1731408845809 (+9 ms)Closed at 1731408845809 2024-11-12T10:54:05,810 WARN [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: f3fbab0ede17db49ead8375d789c7d6a to self. 2024-11-12T10:54:05,812 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:05,813 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f3fbab0ede17db49ead8375d789c7d6a, regionState=CLOSED 2024-11-12T10:54:05,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:54:05,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-12T10:54:05,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 in 171 msec 2024-11-12T10:54:05,820 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, REOPEN/MOVE; state=CLOSED, location=aed202db7571,43143,1731408812834; forceNewPlan=false, retain=true 2024-11-12T10:54:05,971 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f3fbab0ede17db49ead8375d789c7d6a, regionState=OPENING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:54:05,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, REOPEN/MOVE because future has completed 2024-11-12T10:54:05,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:54:06,136 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:06,136 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:54:06,137 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,137 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:54:06,138 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,138 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,146 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,147 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:54:06,152 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fbab0ede17db49ead8375d789c7d6a columnFamilyName A 2024-11-12T10:54:06,154 DEBUG [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:06,155 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.HStore(327): Store=f3fbab0ede17db49ead8375d789c7d6a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:54:06,155 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,156 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:54:06,156 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fbab0ede17db49ead8375d789c7d6a columnFamilyName B 2024-11-12T10:54:06,156 DEBUG [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:06,157 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.HStore(327): Store=f3fbab0ede17db49ead8375d789c7d6a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:54:06,157 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,157 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:54:06,157 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fbab0ede17db49ead8375d789c7d6a columnFamilyName C 2024-11-12T10:54:06,158 DEBUG [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:06,158 INFO [StoreOpener-f3fbab0ede17db49ead8375d789c7d6a-1 {}] regionserver.HStore(327): Store=f3fbab0ede17db49ead8375d789c7d6a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:54:06,158 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:06,158 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,159 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,160 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,161 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,161 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,161 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T10:54:06,162 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,163 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened f3fbab0ede17db49ead8375d789c7d6a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61904946, jitterRate=-0.07754442095756531}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T10:54:06,163 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,164 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for f3fbab0ede17db49ead8375d789c7d6a: Running coprocessor pre-open hook at 1731408846138Writing region info on filesystem at 1731408846138Initializing all the Stores at 1731408846140 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408846140Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408846141 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408846141Cleaning up temporary data from old regions at 1731408846161 (+20 ms)Running coprocessor post-open hooks at 1731408846163 (+2 ms)Region opened successfully at 1731408846164 (+1 ms) 2024-11-12T10:54:06,165 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., pid=35, masterSystemTime=1731408846129 2024-11-12T10:54:06,166 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:06,166 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:06,167 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f3fbab0ede17db49ead8375d789c7d6a, regionState=OPEN, openSeqNum=5, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:54:06,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-12T10:54:06,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 in 192 msec 2024-11-12T10:54:06,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-12T10:54:06,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, REOPEN/MOVE in 530 msec 2024-11-12T10:54:06,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-12T10:54:06,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 542 msec 2024-11-12T10:54:06,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 972 msec 2024-11-12T10:54:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-12T10:54:06,181 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:06,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:06,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:06,183 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:06,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:06,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:06,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e92f053, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:06,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:06,185 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3146dbb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:06,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:06,187 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a87beba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:06,189 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78ec65ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:06,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:06,192 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36433608, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:06,193 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a4b28bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:06,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:06,195 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:06,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:06,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:06,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f6b7f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:06,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:06,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:06,197 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:06,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ceebc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:06,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:06,199 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744bb8aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:06,201 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dda894e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,202 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:06,202 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:06,203 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:06,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:06,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:06,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ec4ca54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:06,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:06,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:06,204 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64972fdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,206 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:06,206 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:06,207 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:06,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:06,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d875f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:06,209 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:06,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@738d24db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,210 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:06,210 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:06,211 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@605febde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:06,213 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:06,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70104d17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:06,215 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:06,216 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55cae0b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:06,218 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ba1929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:06,220 DEBUG [Thread-594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:06,220 DEBUG [Thread-596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:06,220 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:06,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:06,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-12T10:54:06,221 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:06,221 DEBUG [Thread-605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:06,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:06,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:06,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:06,223 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:06,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T10:54:06,224 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:06,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:06,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:06,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:06,226 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:06,226 DEBUG [Thread-588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:06,227 DEBUG [Thread-588 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:06,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:54:06,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:54:06,228 DEBUG [Thread-588 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:06,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:06,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:54:06,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:06,230 DEBUG [Thread-590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:06,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:06,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:54:06,232 DEBUG [Thread-592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:06,233 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:54:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:06,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:54:06,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:06,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:06,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:54:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:54:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:54:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T10:54:06,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:06,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:06,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:06,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:06,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:06,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:06,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111262915d1649b94af0a0895fe718b11725_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408846244/Put/seqid=0 2024-11-12T10:54:06,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741939_1115 (size=14594) 2024-11-12T10:54:06,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408906295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408906297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408906300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408906301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1731408846271, max timestamp=1731408846315Num uniques 859; 2024-11-12T10:54:06,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-12T10:54:06,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1731408846271, max timestamp=1731408846315Num uniques 859; 2024-11-12T10:54:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T10:54:06,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-12T10:54:06,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1731408846271, max timestamp=1731408846315Num uniques 859; 2024-11-12T10:54:06,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-12T10:54:06,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T10:54:06,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:06,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:06,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:06,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:06,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:06,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408906427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408906427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408906428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408906428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408906432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T10:54:06,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:06,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:06,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:06,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T10:54:06,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408906540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408906637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408906637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408906637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408906637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T10:54:06,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:06,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:06,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:06,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:06,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:06,715 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:06,721 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111262915d1649b94af0a0895fe718b11725_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111262915d1649b94af0a0895fe718b11725_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:06,722 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/1813b698abac442db5a79dfce2f6a969, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:06,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/1813b698abac442db5a79dfce2f6a969 is 175, key is test_row_0/A:col10/1731408846244/Put/seqid=0 2024-11-12T10:54:06,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741940_1116 (size=39549) 2024-11-12T10:54:06,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408906758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T10:54:06,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:06,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:06,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:06,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:06,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T10:54:06,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408906947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408906948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408906948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:06,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408906948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:06,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T10:54:06,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:06,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:06,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:06,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:06,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:07,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408907077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:07,146 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/1813b698abac442db5a79dfce2f6a969 2024-11-12T10:54:07,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T10:54:07,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:07,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:07,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:07,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:07,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/ebfa6f0ccaba461e8d3d1a03d72b0c36 is 50, key is test_row_0/B:col10/1731408846244/Put/seqid=0 2024-11-12T10:54:07,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741941_1117 (size=12001) 2024-11-12T10:54:07,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/ebfa6f0ccaba461e8d3d1a03d72b0c36 2024-11-12T10:54:07,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/2746ded2bac549268428738000f01db4 is 50, key is test_row_0/C:col10/1731408846244/Put/seqid=0 2024-11-12T10:54:07,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741942_1118 (size=12001) 2024-11-12T10:54:07,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/2746ded2bac549268428738000f01db4 2024-11-12T10:54:07,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/1813b698abac442db5a79dfce2f6a969 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/1813b698abac442db5a79dfce2f6a969 2024-11-12T10:54:07,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/1813b698abac442db5a79dfce2f6a969, entries=200, sequenceid=15, filesize=38.6 K 2024-11-12T10:54:07,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/ebfa6f0ccaba461e8d3d1a03d72b0c36 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/ebfa6f0ccaba461e8d3d1a03d72b0c36 2024-11-12T10:54:07,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/ebfa6f0ccaba461e8d3d1a03d72b0c36, entries=150, sequenceid=15, filesize=11.7 K 2024-11-12T10:54:07,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/2746ded2bac549268428738000f01db4 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/2746ded2bac549268428738000f01db4 2024-11-12T10:54:07,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/2746ded2bac549268428738000f01db4, entries=150, sequenceid=15, filesize=11.7 K 2024-11-12T10:54:07,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for f3fbab0ede17db49ead8375d789c7d6a in 1050ms, sequenceid=15, compaction requested=false 2024-11-12T10:54:07,296 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T10:54:07,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:07,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T10:54:07,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:07,309 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=134.18 KB heapSize=352.17 KB 2024-11-12T10:54:07,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:07,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:07,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:07,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:07,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:07,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:07,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127f81ae60d0544ba5b02fc98d1e9791d7_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408846398/Put/seqid=0 2024-11-12T10:54:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T10:54:07,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741943_1119 (size=12154) 2024-11-12T10:54:07,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:07,379 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127f81ae60d0544ba5b02fc98d1e9791d7_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127f81ae60d0544ba5b02fc98d1e9791d7_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:07,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/53bc9a10402941ec859fccbd0762a5a5, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:07,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/53bc9a10402941ec859fccbd0762a5a5 is 175, key is test_row_0/A:col10/1731408846398/Put/seqid=0 2024-11-12T10:54:07,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741944_1120 (size=30955) 2024-11-12T10:54:07,415 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=45, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/53bc9a10402941ec859fccbd0762a5a5 2024-11-12T10:54:07,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/1d13a1fb26554814946442cd97eb7f13 is 50, key is test_row_0/B:col10/1731408846398/Put/seqid=0 2024-11-12T10:54:07,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741945_1121 (size=12001) 2024-11-12T10:54:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:07,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:07,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408907473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:07,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408907476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:07,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408907479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:07,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408907479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:07,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:07,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408907587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:07,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408907588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408907588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:07,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408907588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:07,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408907588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:07,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:07,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408907797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:07,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408907798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:07,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:07,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408907798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:07,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:07,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:07,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408907798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:07,858 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/1d13a1fb26554814946442cd97eb7f13 2024-11-12T10:54:07,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/8cc57cc485d843e9a7ffc58e0337cea7 is 50, key is test_row_0/C:col10/1731408846398/Put/seqid=0 2024-11-12T10:54:07,872 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T10:54:07,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741946_1122 (size=12001) 2024-11-12T10:54:08,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408908107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408908107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408908107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408908108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,304 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/8cc57cc485d843e9a7ffc58e0337cea7 2024-11-12T10:54:08,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/53bc9a10402941ec859fccbd0762a5a5 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/53bc9a10402941ec859fccbd0762a5a5 2024-11-12T10:54:08,317 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/53bc9a10402941ec859fccbd0762a5a5, entries=150, sequenceid=45, filesize=30.2 K 2024-11-12T10:54:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/1d13a1fb26554814946442cd97eb7f13 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/1d13a1fb26554814946442cd97eb7f13 2024-11-12T10:54:08,328 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/1d13a1fb26554814946442cd97eb7f13, entries=150, sequenceid=45, filesize=11.7 K 2024-11-12T10:54:08,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/8cc57cc485d843e9a7ffc58e0337cea7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/8cc57cc485d843e9a7ffc58e0337cea7 2024-11-12T10:54:08,342 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/8cc57cc485d843e9a7ffc58e0337cea7, entries=150, sequenceid=45, filesize=11.7 K 2024-11-12T10:54:08,344 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.13 KB/360576, currentSize=60.38 KB/61830 for f3fbab0ede17db49ead8375d789c7d6a in 1034ms, sequenceid=45, compaction requested=false 2024-11-12T10:54:08,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:08,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:08,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-12T10:54:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-12T10:54:08,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-12T10:54:08,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1210 sec 2024-11-12T10:54:08,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.1280 sec 2024-11-12T10:54:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T10:54:08,367 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:08,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-12T10:54:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T10:54:08,370 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:08,371 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:08,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T10:54:08,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T10:54:08,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:08,525 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T10:54:08,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:08,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:08,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:08,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:08,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:08,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:08,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125ea871b61c334fa18d7388439a308171_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408847475/Put/seqid=0 2024-11-12T10:54:08,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741947_1123 (size=12154) 2024-11-12T10:54:08,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:08,570 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125ea871b61c334fa18d7388439a308171_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125ea871b61c334fa18d7388439a308171_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:08,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/f894226f031144158a40c6655b40966f, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/f894226f031144158a40c6655b40966f is 175, key is test_row_0/A:col10/1731408847475/Put/seqid=0 2024-11-12T10:54:08,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741948_1124 (size=30955) 2024-11-12T10:54:08,586 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=57, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/f894226f031144158a40c6655b40966f 2024-11-12T10:54:08,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/e7478eba89524deeb03e5a1454e94eb5 is 50, key is test_row_0/B:col10/1731408847475/Put/seqid=0 2024-11-12T10:54:08,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:08,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741949_1125 (size=12001) 2024-11-12T10:54:08,627 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/e7478eba89524deeb03e5a1454e94eb5 2024-11-12T10:54:08,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/81094d7781394eb9bff03bf360e57508 is 50, key is test_row_0/C:col10/1731408847475/Put/seqid=0 2024-11-12T10:54:08,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408908642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408908643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408908646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408908653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741950_1126 (size=12001) 2024-11-12T10:54:08,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408908656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T10:54:08,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408908757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408908767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408908767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408908767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408908767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408908967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408908977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408908978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408908978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408908978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T10:54:09,060 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/81094d7781394eb9bff03bf360e57508 2024-11-12T10:54:09,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/f894226f031144158a40c6655b40966f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/f894226f031144158a40c6655b40966f 2024-11-12T10:54:09,071 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/f894226f031144158a40c6655b40966f, entries=150, sequenceid=57, filesize=30.2 K 2024-11-12T10:54:09,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/e7478eba89524deeb03e5a1454e94eb5 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/e7478eba89524deeb03e5a1454e94eb5 2024-11-12T10:54:09,077 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/e7478eba89524deeb03e5a1454e94eb5, entries=150, sequenceid=57, filesize=11.7 K 2024-11-12T10:54:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/81094d7781394eb9bff03bf360e57508 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/81094d7781394eb9bff03bf360e57508 2024-11-12T10:54:09,083 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/81094d7781394eb9bff03bf360e57508, entries=150, sequenceid=57, filesize=11.7 K 2024-11-12T10:54:09,084 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f3fbab0ede17db49ead8375d789c7d6a in 559ms, sequenceid=57, compaction requested=true 2024-11-12T10:54:09,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:09,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:09,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-12T10:54:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-12T10:54:09,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-12T10:54:09,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 715 msec 2024-11-12T10:54:09,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 721 msec 2024-11-12T10:54:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:09,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408848636, max timestamp=1731408849278Num uniques 859; 2024-11-12T10:54:09,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408848636, max timestamp=1731408849278Num uniques 859; 2024-11-12T10:54:09,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408848636, max timestamp=1731408849278Num uniques 859; 2024-11-12T10:54:09,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:54:09,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:54:09,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-12T10:54:09,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:09,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:54:09,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:09,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:09,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:09,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:09,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:09,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111233fd7fbfbe5c46198a6d06e07542b22d_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408849278/Put/seqid=0 2024-11-12T10:54:09,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731408849290, max timestamp=1731408849310Num uniques 859; 2024-11-12T10:54:09,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-12T10:54:09,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731408849290, max timestamp=1731408849310Num uniques 859; 2024-11-12T10:54:09,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-12T10:54:09,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731408849290, max timestamp=1731408849310Num uniques 859; 2024-11-12T10:54:09,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-12T10:54:09,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741951_1127 (size=14594) 2024-11-12T10:54:09,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408909331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408909346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408909346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408909347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408909347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408909457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408909457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408909457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408909457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408909457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T10:54:09,507 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:09,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-12T10:54:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T10:54:09,509 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:09,510 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:09,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T10:54:09,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T10:54:09,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:09,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:09,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:09,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:09,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:09,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408909667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408909667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408909668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408909668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408909668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,735 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:09,739 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111233fd7fbfbe5c46198a6d06e07542b22d_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111233fd7fbfbe5c46198a6d06e07542b22d_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:09,740 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/b036afe4c3e24b11a3e30071d0d31261, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:09,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/b036afe4c3e24b11a3e30071d0d31261 is 175, key is test_row_0/A:col10/1731408849278/Put/seqid=0 2024-11-12T10:54:09,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741952_1128 (size=39549) 2024-11-12T10:54:09,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T10:54:09,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:09,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:09,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:09,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:09,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:09,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T10:54:09,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T10:54:09,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:09,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:09,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:09,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:09,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:09,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408909977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408909977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408909977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408909977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408909977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:10,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T10:54:10,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:10,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:10,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:10,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T10:54:10,147 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/b036afe4c3e24b11a3e30071d0d31261 2024-11-12T10:54:10,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/b0369996b6fe4b3e91a0179bc6961512 is 50, key is test_row_0/B:col10/1731408849278/Put/seqid=0 2024-11-12T10:54:10,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741953_1129 (size=12001) 2024-11-12T10:54:10,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T10:54:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:10,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T10:54:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:10,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408910488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:10,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:10,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408910488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:10,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408910488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:10,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408910488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:10,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408910488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:10,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/b0369996b6fe4b3e91a0179bc6961512 2024-11-12T10:54:10,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/5250d73d88594b8a8e77fbdd35d1e0b2 is 50, key is test_row_0/C:col10/1731408849278/Put/seqid=0 2024-11-12T10:54:10,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741954_1130 (size=12001) 2024-11-12T10:54:10,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T10:54:10,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:10,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:10,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:10,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T10:54:10,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T10:54:10,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:10,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:10,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:10,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T10:54:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:10,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:10,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/5250d73d88594b8a8e77fbdd35d1e0b2 2024-11-12T10:54:10,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/b036afe4c3e24b11a3e30071d0d31261 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/b036afe4c3e24b11a3e30071d0d31261 2024-11-12T10:54:10,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/b036afe4c3e24b11a3e30071d0d31261, entries=200, sequenceid=78, filesize=38.6 K 2024-11-12T10:54:10,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/b0369996b6fe4b3e91a0179bc6961512 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/b0369996b6fe4b3e91a0179bc6961512 2024-11-12T10:54:11,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/b0369996b6fe4b3e91a0179bc6961512, entries=150, sequenceid=78, filesize=11.7 K 2024-11-12T10:54:11,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/5250d73d88594b8a8e77fbdd35d1e0b2 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/5250d73d88594b8a8e77fbdd35d1e0b2 2024-11-12T10:54:11,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/5250d73d88594b8a8e77fbdd35d1e0b2, entries=150, sequenceid=78, filesize=11.7 K 2024-11-12T10:54:11,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=167.72 KB/171750 for f3fbab0ede17db49ead8375d789c7d6a in 1728ms, sequenceid=78, compaction requested=true 2024-11-12T10:54:11,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:11,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:11,010 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:11,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:11,010 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:11,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:11,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:11,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:11,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:11,011 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:11,011 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:11,011 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/A is initiating minor compaction (all files) 2024-11-12T10:54:11,011 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/B is initiating minor compaction (all files) 2024-11-12T10:54:11,012 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/B in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:11,012 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/A in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:11,012 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/ebfa6f0ccaba461e8d3d1a03d72b0c36, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/1d13a1fb26554814946442cd97eb7f13, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/e7478eba89524deeb03e5a1454e94eb5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/b0369996b6fe4b3e91a0179bc6961512] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=46.9 K 2024-11-12T10:54:11,012 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/1813b698abac442db5a79dfce2f6a969, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/53bc9a10402941ec859fccbd0762a5a5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/f894226f031144158a40c6655b40966f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/b036afe4c3e24b11a3e30071d0d31261] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=137.7 K 2024-11-12T10:54:11,012 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ebfa6f0ccaba461e8d3d1a03d72b0c36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731408846239 2024-11-12T10:54:11,012 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:11,012 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/1813b698abac442db5a79dfce2f6a969, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/53bc9a10402941ec859fccbd0762a5a5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/f894226f031144158a40c6655b40966f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/b036afe4c3e24b11a3e30071d0d31261] 2024-11-12T10:54:11,012 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d13a1fb26554814946442cd97eb7f13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731408846299 2024-11-12T10:54:11,012 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1813b698abac442db5a79dfce2f6a969, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731408846239 2024-11-12T10:54:11,013 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting e7478eba89524deeb03e5a1454e94eb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731408847471 2024-11-12T10:54:11,013 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53bc9a10402941ec859fccbd0762a5a5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731408846299 2024-11-12T10:54:11,013 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting b0369996b6fe4b3e91a0179bc6961512, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731408848636 2024-11-12T10:54:11,013 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting f894226f031144158a40c6655b40966f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731408847471 2024-11-12T10:54:11,016 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting b036afe4c3e24b11a3e30071d0d31261, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731408848636 2024-11-12T10:54:11,024 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#B#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:11,026 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/40e2d961b1c1432d90faeb7ce5fd22ff is 50, key is test_row_0/B:col10/1731408849278/Put/seqid=0 2024-11-12T10:54:11,027 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:11,030 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112974ef67fd6144ec0a650e429752f41be_f3fbab0ede17db49ead8375d789c7d6a store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:11,036 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112974ef67fd6144ec0a650e429752f41be_f3fbab0ede17db49ead8375d789c7d6a, store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:11,036 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112974ef67fd6144ec0a650e429752f41be_f3fbab0ede17db49ead8375d789c7d6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:11,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T10:54:11,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:11,046 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=167.72 KB heapSize=456.47 KB 2024-11-12T10:54:11,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:11,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:11,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:11,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:11,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:11,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:11,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741955_1131 (size=12139) 2024-11-12T10:54:11,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741956_1132 (size=4469) 2024-11-12T10:54:11,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c167b4a082274a4f8ebe357bef9ef4ea_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408849344/Put/seqid=0 2024-11-12T10:54:11,082 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/40e2d961b1c1432d90faeb7ce5fd22ff as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/40e2d961b1c1432d90faeb7ce5fd22ff 2024-11-12T10:54:11,089 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/B of f3fbab0ede17db49ead8375d789c7d6a into 40e2d961b1c1432d90faeb7ce5fd22ff(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:11,089 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:11,089 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/B, priority=12, startTime=1731408851010; duration=0sec 2024-11-12T10:54:11,089 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:11,089 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:B 2024-11-12T10:54:11,089 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:11,091 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:11,091 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/C is initiating minor compaction (all files) 2024-11-12T10:54:11,091 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/C in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:11,091 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/2746ded2bac549268428738000f01db4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/8cc57cc485d843e9a7ffc58e0337cea7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/81094d7781394eb9bff03bf360e57508, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/5250d73d88594b8a8e77fbdd35d1e0b2] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=46.9 K 2024-11-12T10:54:11,092 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 2746ded2bac549268428738000f01db4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731408846239 2024-11-12T10:54:11,092 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cc57cc485d843e9a7ffc58e0337cea7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731408846299 2024-11-12T10:54:11,092 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 81094d7781394eb9bff03bf360e57508, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731408847471 2024-11-12T10:54:11,092 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 5250d73d88594b8a8e77fbdd35d1e0b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731408848636 2024-11-12T10:54:11,110 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#C#compaction#114 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:11,110 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/9b93b45648d04d74b7b2ed4ed981522a is 50, key is test_row_0/C:col10/1731408849278/Put/seqid=0 2024-11-12T10:54:11,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741957_1133 (size=12154) 2024-11-12T10:54:11,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:11,117 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c167b4a082274a4f8ebe357bef9ef4ea_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c167b4a082274a4f8ebe357bef9ef4ea_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:11,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/890be27b4aa047559653d99f9dd3df4e, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:11,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/890be27b4aa047559653d99f9dd3df4e is 175, key is test_row_0/A:col10/1731408849344/Put/seqid=0 2024-11-12T10:54:11,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741959_1135 (size=30955) 2024-11-12T10:54:11,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741958_1134 (size=12139) 2024-11-12T10:54:11,474 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#A#compaction#112 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:11,475 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/afda5678fff1499a8f923d37736d1904 is 175, key is test_row_0/A:col10/1731408849278/Put/seqid=0 2024-11-12T10:54:11,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741960_1136 (size=31093) 2024-11-12T10:54:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:11,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408911499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408911500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:11,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408911501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:11,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408911507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:11,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408911507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:11,514 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in f3fbab0ede17db49ead8375d789c7d6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,515 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in f3fbab0ede17db49ead8375d789c7d6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:11,527 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/890be27b4aa047559653d99f9dd3df4e 2024-11-12T10:54:11,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/adcd16868df446ed81f9e1799e45517a is 50, key is test_row_0/B:col10/1731408849344/Put/seqid=0 2024-11-12T10:54:11,536 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/9b93b45648d04d74b7b2ed4ed981522a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/9b93b45648d04d74b7b2ed4ed981522a 2024-11-12T10:54:11,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741961_1137 (size=12001) 2024-11-12T10:54:11,542 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/C of f3fbab0ede17db49ead8375d789c7d6a into 9b93b45648d04d74b7b2ed4ed981522a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:11,542 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:11,542 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/C, priority=12, startTime=1731408851010; duration=0sec 2024-11-12T10:54:11,542 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:11,543 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:C 2024-11-12T10:54:11,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:11,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408911607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:11,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:11,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:11,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408911617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:11,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:11,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408911617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:11,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:11,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T10:54:11,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408911817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:11,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408911827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:11,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408911827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:11,892 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/afda5678fff1499a8f923d37736d1904 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/afda5678fff1499a8f923d37736d1904 2024-11-12T10:54:11,900 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/A of f3fbab0ede17db49ead8375d789c7d6a into afda5678fff1499a8f923d37736d1904(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:11,900 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:11,900 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/A, priority=12, startTime=1731408851009; duration=0sec 2024-11-12T10:54:11,900 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:11,900 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:A 2024-11-12T10:54:11,941 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/adcd16868df446ed81f9e1799e45517a 2024-11-12T10:54:11,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/d5415c0d2927497a823211647e22cb44 is 50, key is test_row_0/C:col10/1731408849344/Put/seqid=0 2024-11-12T10:54:11,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741962_1138 (size=12001) 2024-11-12T10:54:12,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408912127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:12,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408912136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:12,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:12,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408912137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:12,360 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/d5415c0d2927497a823211647e22cb44 2024-11-12T10:54:12,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/890be27b4aa047559653d99f9dd3df4e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/890be27b4aa047559653d99f9dd3df4e 2024-11-12T10:54:12,369 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/890be27b4aa047559653d99f9dd3df4e, entries=150, sequenceid=114, filesize=30.2 K 2024-11-12T10:54:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/adcd16868df446ed81f9e1799e45517a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/adcd16868df446ed81f9e1799e45517a 2024-11-12T10:54:12,378 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/adcd16868df446ed81f9e1799e45517a, entries=150, sequenceid=114, filesize=11.7 K 2024-11-12T10:54:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/d5415c0d2927497a823211647e22cb44 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/d5415c0d2927497a823211647e22cb44 2024-11-12T10:54:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,387 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/d5415c0d2927497a823211647e22cb44, entries=150, sequenceid=114, filesize=11.7 K 2024-11-12T10:54:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,389 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~456.42 KB/467376, currentSize=20.13 KB/20610 for f3fbab0ede17db49ead8375d789c7d6a in 1342ms, sequenceid=114, compaction requested=false 2024-11-12T10:54:12,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:12,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:12,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-12T10:54:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-12T10:54:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-12T10:54:12,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8800 sec 2024-11-12T10:54:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.8870 sec 2024-11-12T10:54:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T10:54:12,591 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-12T10:54:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T10:54:12,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:12,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:12,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:12,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:12,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a8bebc94745f412aab98aaa4552595c7_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408852651/Put/seqid=0 2024-11-12T10:54:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741963_1139 (size=19474) 2024-11-12T10:54:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,677 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:12,683 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a8bebc94745f412aab98aaa4552595c7_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a8bebc94745f412aab98aaa4552595c7_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:12,685 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/037fba659c00423fab261a762e390d07, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:12,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/037fba659c00423fab261a762e390d07 is 175, key is test_row_0/A:col10/1731408852651/Put/seqid=0 2024-11-12T10:54:12,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741964_1140 (size=56733) 2024-11-12T10:54:12,700 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/037fba659c00423fab261a762e390d07 2024-11-12T10:54:12,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731408852698, max timestamp=1731408852707Num uniques 859; 2024-11-12T10:54:12,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T10:54:12,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731408852698, max timestamp=1731408852707Num uniques 859; 2024-11-12T10:54:12,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731408852698, max timestamp=1731408852707Num uniques 859; 2024-11-12T10:54:12,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T10:54:12,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T10:54:12,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/284aaea047974f758b7d2a108b43cef7 is 50, key is test_row_0/B:col10/1731408852651/Put/seqid=0 2024-11-12T10:54:12,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741965_1141 (size=12001) 2024-11-12T10:54:12,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:12,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408912767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:12,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408912769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:12,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:12,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408912769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:12,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408912887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:12,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408912887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:12,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408912887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:13,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:13,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408913097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:13,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408913098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:13,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408913098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:13,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/284aaea047974f758b7d2a108b43cef7 2024-11-12T10:54:13,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ce7662d41d354a6bb5dfe86f7c5f9b5e is 50, key is test_row_0/C:col10/1731408852651/Put/seqid=0 2024-11-12T10:54:13,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741966_1142 (size=12001) 2024-11-12T10:54:13,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408913407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:13,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:13,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408913407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:13,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:13,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408913407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:13,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:13,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408913528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:13,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408913528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:13,530 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in f3fbab0ede17db49ead8375d789c7d6a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:13,530 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in f3fbab0ede17db49ead8375d789c7d6a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:13,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ce7662d41d354a6bb5dfe86f7c5f9b5e 2024-11-12T10:54:13,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/037fba659c00423fab261a762e390d07 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/037fba659c00423fab261a762e390d07 2024-11-12T10:54:13,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/037fba659c00423fab261a762e390d07, entries=300, sequenceid=127, filesize=55.4 K 2024-11-12T10:54:13,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/284aaea047974f758b7d2a108b43cef7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/284aaea047974f758b7d2a108b43cef7 2024-11-12T10:54:13,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/284aaea047974f758b7d2a108b43cef7, entries=150, sequenceid=127, filesize=11.7 K 2024-11-12T10:54:13,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ce7662d41d354a6bb5dfe86f7c5f9b5e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ce7662d41d354a6bb5dfe86f7c5f9b5e 2024-11-12T10:54:13,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ce7662d41d354a6bb5dfe86f7c5f9b5e, entries=150, sequenceid=127, filesize=11.7 K 2024-11-12T10:54:13,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f3fbab0ede17db49ead8375d789c7d6a in 927ms, sequenceid=127, compaction requested=true 2024-11-12T10:54:13,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:13,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:13,581 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:13,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:13,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:13,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:13,581 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:13,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:13,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:13,585 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118781 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:13,585 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/A is initiating minor compaction (all files) 2024-11-12T10:54:13,585 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/A in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:13,585 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/afda5678fff1499a8f923d37736d1904, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/890be27b4aa047559653d99f9dd3df4e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/037fba659c00423fab261a762e390d07] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=116.0 K 2024-11-12T10:54:13,585 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:13,586 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/afda5678fff1499a8f923d37736d1904, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/890be27b4aa047559653d99f9dd3df4e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/037fba659c00423fab261a762e390d07] 2024-11-12T10:54:13,586 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:13,586 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting afda5678fff1499a8f923d37736d1904, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731408848636 2024-11-12T10:54:13,586 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/B is initiating minor compaction (all files) 2024-11-12T10:54:13,586 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/B in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:13,586 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/40e2d961b1c1432d90faeb7ce5fd22ff, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/adcd16868df446ed81f9e1799e45517a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/284aaea047974f758b7d2a108b43cef7] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=35.3 K 2024-11-12T10:54:13,586 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 890be27b4aa047559653d99f9dd3df4e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731408849344 2024-11-12T10:54:13,587 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 40e2d961b1c1432d90faeb7ce5fd22ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731408848636 2024-11-12T10:54:13,587 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 037fba659c00423fab261a762e390d07, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731408851499 2024-11-12T10:54:13,587 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting adcd16868df446ed81f9e1799e45517a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731408849344 2024-11-12T10:54:13,587 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 284aaea047974f758b7d2a108b43cef7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731408852644 2024-11-12T10:54:13,600 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#B#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:13,601 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/7ba992eba3344a46af01f32012bb6670 is 50, key is test_row_0/B:col10/1731408852651/Put/seqid=0 2024-11-12T10:54:13,605 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:13,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741967_1143 (size=12241) 2024-11-12T10:54:13,619 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411127c4b6fdeba544e8cb1f603b4b514c53a_f3fbab0ede17db49ead8375d789c7d6a store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:13,622 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411127c4b6fdeba544e8cb1f603b4b514c53a_f3fbab0ede17db49ead8375d789c7d6a, store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:13,622 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127c4b6fdeba544e8cb1f603b4b514c53a_f3fbab0ede17db49ead8375d789c7d6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:13,627 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/7ba992eba3344a46af01f32012bb6670 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/7ba992eba3344a46af01f32012bb6670 2024-11-12T10:54:13,640 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/B of f3fbab0ede17db49ead8375d789c7d6a into 7ba992eba3344a46af01f32012bb6670(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:13,640 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:13,640 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/B, priority=13, startTime=1731408853581; duration=0sec 2024-11-12T10:54:13,640 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:13,640 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:B 2024-11-12T10:54:13,641 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:13,643 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:13,643 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/C is initiating minor compaction (all files) 2024-11-12T10:54:13,643 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/C in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:13,643 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/9b93b45648d04d74b7b2ed4ed981522a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/d5415c0d2927497a823211647e22cb44, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ce7662d41d354a6bb5dfe86f7c5f9b5e] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=35.3 K 2024-11-12T10:54:13,644 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b93b45648d04d74b7b2ed4ed981522a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731408848636 2024-11-12T10:54:13,644 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting d5415c0d2927497a823211647e22cb44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731408849344 2024-11-12T10:54:13,644 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ce7662d41d354a6bb5dfe86f7c5f9b5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731408852644 2024-11-12T10:54:13,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741968_1144 (size=4469) 2024-11-12T10:54:13,648 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#A#compaction#121 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:13,648 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/c1518a51407148baa4ede524df352189 is 175, key is test_row_0/A:col10/1731408852651/Put/seqid=0 2024-11-12T10:54:13,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741969_1145 (size=31195) 2024-11-12T10:54:13,664 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#C#compaction#122 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:13,665 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/3af8c4591f224263b9235fbf0c93ff57 is 50, key is test_row_0/C:col10/1731408852651/Put/seqid=0 2024-11-12T10:54:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T10:54:13,667 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:13,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-12T10:54:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T10:54:13,671 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:13,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741970_1146 (size=12241) 2024-11-12T10:54:13,673 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:13,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:13,688 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/3af8c4591f224263b9235fbf0c93ff57 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/3af8c4591f224263b9235fbf0c93ff57 2024-11-12T10:54:13,696 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/C of f3fbab0ede17db49ead8375d789c7d6a into 3af8c4591f224263b9235fbf0c93ff57(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:13,696 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:13,696 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/C, priority=13, startTime=1731408853581; duration=0sec 2024-11-12T10:54:13,697 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:13,697 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:C 2024-11-12T10:54:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T10:54:13,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T10:54:13,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:13,837 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-12T10:54:13,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:13,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:13,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:13,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:13,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:13,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:13,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112eeac0b7d219841258abe539c6999ee73_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408852766/Put/seqid=0 2024-11-12T10:54:13,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741971_1147 (size=12304) 2024-11-12T10:54:13,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:13,862 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112eeac0b7d219841258abe539c6999ee73_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112eeac0b7d219841258abe539c6999ee73_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:13,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/71acf022f4d748d18083038373a392b8, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:13,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/71acf022f4d748d18083038373a392b8 is 175, key is test_row_0/A:col10/1731408852766/Put/seqid=0 2024-11-12T10:54:13,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741972_1148 (size=31105) 2024-11-12T10:54:13,868 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/71acf022f4d748d18083038373a392b8 2024-11-12T10:54:13,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/9889f7efbda84420a37d92f9070ee6d7 is 50, key is test_row_0/B:col10/1731408852766/Put/seqid=0 2024-11-12T10:54:13,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741973_1149 (size=12151) 2024-11-12T10:54:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:13,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:13,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:13,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408913928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:13,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408913929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:13,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:13,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408913930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T10:54:14,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:14,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408914036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408914037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:14,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408914047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:14,064 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/c1518a51407148baa4ede524df352189 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/c1518a51407148baa4ede524df352189 2024-11-12T10:54:14,070 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/A of f3fbab0ede17db49ead8375d789c7d6a into c1518a51407148baa4ede524df352189(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:14,070 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:14,070 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/A, priority=13, startTime=1731408853581; duration=0sec 2024-11-12T10:54:14,070 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:14,070 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:A 2024-11-12T10:54:14,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408914246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:14,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408914246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:14,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408914257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:14,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:14,285 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/9889f7efbda84420a37d92f9070ee6d7 2024-11-12T10:54:14,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ebc68785ed2940cd972266751f72690a is 50, key is test_row_0/C:col10/1731408852766/Put/seqid=0 2024-11-12T10:54:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T10:54:14,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741974_1150 (size=12151) 2024-11-12T10:54:14,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:14,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408914557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408914557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:14,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408914567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:14,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:14,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:14,699 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ebc68785ed2940cd972266751f72690a 2024-11-12T10:54:14,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/71acf022f4d748d18083038373a392b8 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/71acf022f4d748d18083038373a392b8 2024-11-12T10:54:14,710 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/71acf022f4d748d18083038373a392b8, entries=150, sequenceid=165, filesize=30.4 K 2024-11-12T10:54:14,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/9889f7efbda84420a37d92f9070ee6d7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/9889f7efbda84420a37d92f9070ee6d7 2024-11-12T10:54:14,715 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/9889f7efbda84420a37d92f9070ee6d7, entries=150, sequenceid=165, filesize=11.9 K 2024-11-12T10:54:14,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ebc68785ed2940cd972266751f72690a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ebc68785ed2940cd972266751f72690a 2024-11-12T10:54:14,721 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ebc68785ed2940cd972266751f72690a, entries=150, sequenceid=165, filesize=11.9 K 2024-11-12T10:54:14,722 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for f3fbab0ede17db49ead8375d789c7d6a in 885ms, sequenceid=165, compaction requested=false 2024-11-12T10:54:14,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:14,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:14,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-12T10:54:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-12T10:54:14,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-12T10:54:14,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0510 sec 2024-11-12T10:54:14,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.0580 sec 2024-11-12T10:54:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T10:54:14,807 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:14,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-12T10:54:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T10:54:14,809 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:14,810 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:14,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T10:54:14,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T10:54:14,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:14,963 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-12T10:54:14,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:14,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:14,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:14,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:14,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:14,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:14,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111290127d1678464a568974a0d967b1a3ab_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408853928/Put/seqid=0 2024-11-12T10:54:14,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741975_1151 (size=12304) 2024-11-12T10:54:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:15,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:15,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731408855087, max timestamp=1731408855102Num uniques 859; 2024-11-12T10:54:15,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731408855087, max timestamp=1731408855102Num uniques 859; 2024-11-12T10:54:15,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T10:54:15,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T10:54:15,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731408855087, max timestamp=1731408855102Num uniques 859; 2024-11-12T10:54:15,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T10:54:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T10:54:15,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408915128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:15,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408915129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:15,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408915131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:15,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408915236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:15,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:15,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408915236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408915237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:15,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:15,379 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111290127d1678464a568974a0d967b1a3ab_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111290127d1678464a568974a0d967b1a3ab_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/eacbd119f0ed48c2a4d37289b26c17dd, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:15,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/eacbd119f0ed48c2a4d37289b26c17dd is 175, key is test_row_0/A:col10/1731408853928/Put/seqid=0 2024-11-12T10:54:15,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741976_1152 (size=31105) 2024-11-12T10:54:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T10:54:15,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408915447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:15,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408915447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:15,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:15,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408915448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:15,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:15,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:15,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408915757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:15,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408915757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:15,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408915757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:15,790 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/eacbd119f0ed48c2a4d37289b26c17dd 2024-11-12T10:54:15,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/344ffe34d0304aad89d579e40b846fc6 is 50, key is test_row_0/B:col10/1731408853928/Put/seqid=0 2024-11-12T10:54:15,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741977_1153 (size=12151) 2024-11-12T10:54:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T10:54:16,203 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/344ffe34d0304aad89d579e40b846fc6 2024-11-12T10:54:16,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/2e6a10f10d934528a2270d460e88eb2d is 50, key is test_row_0/C:col10/1731408853928/Put/seqid=0 2024-11-12T10:54:16,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741978_1154 (size=12151) 2024-11-12T10:54:16,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408916267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:16,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408916267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:16,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408916268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:16,614 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/2e6a10f10d934528a2270d460e88eb2d 2024-11-12T10:54:16,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/eacbd119f0ed48c2a4d37289b26c17dd as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/eacbd119f0ed48c2a4d37289b26c17dd 2024-11-12T10:54:16,632 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/eacbd119f0ed48c2a4d37289b26c17dd, entries=150, sequenceid=175, filesize=30.4 K 2024-11-12T10:54:16,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/344ffe34d0304aad89d579e40b846fc6 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/344ffe34d0304aad89d579e40b846fc6 2024-11-12T10:54:16,641 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/344ffe34d0304aad89d579e40b846fc6, entries=150, sequenceid=175, filesize=11.9 K 2024-11-12T10:54:16,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/2e6a10f10d934528a2270d460e88eb2d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/2e6a10f10d934528a2270d460e88eb2d 2024-11-12T10:54:16,647 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/2e6a10f10d934528a2270d460e88eb2d, entries=150, sequenceid=175, filesize=11.9 K 2024-11-12T10:54:16,648 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for f3fbab0ede17db49ead8375d789c7d6a in 1686ms, sequenceid=175, compaction requested=true 2024-11-12T10:54:16,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:16,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:16,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-12T10:54:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-12T10:54:16,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-12T10:54:16,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8410 sec 2024-11-12T10:54:16,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.8500 sec 2024-11-12T10:54:16,750 INFO [master/aed202db7571:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-12T10:54:16,750 INFO [master/aed202db7571:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-12T10:54:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T10:54:16,958 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:16,958 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-12T10:54:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T10:54:16,960 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:16,961 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:16,961 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T10:54:17,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T10:54:17,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:17,113 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-12T10:54:17,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:17,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:17,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:17,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:17,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:17,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d3f7d89270704177bbac60828065e9ed_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408855129/Put/seqid=0 2024-11-12T10:54:17,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741979_1155 (size=12304) 2024-11-12T10:54:17,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:17,131 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d3f7d89270704177bbac60828065e9ed_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d3f7d89270704177bbac60828065e9ed_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:17,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/b059d1e261ff48c590d8d2c7cff2b0da, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:17,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/b059d1e261ff48c590d8d2c7cff2b0da is 175, key is test_row_0/A:col10/1731408855129/Put/seqid=0 2024-11-12T10:54:17,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741980_1156 (size=31105) 2024-11-12T10:54:17,138 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/b059d1e261ff48c590d8d2c7cff2b0da 2024-11-12T10:54:17,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/3634c087ec4942d9b5dd4c64717ebd1f is 50, key is test_row_0/B:col10/1731408855129/Put/seqid=0 2024-11-12T10:54:17,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741981_1157 (size=12151) 2024-11-12T10:54:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T10:54:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:17,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:17,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408917293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:17,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408917297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:17,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408917298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:17,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408917407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:17,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408917407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:17,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408917407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:17,553 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/3634c087ec4942d9b5dd4c64717ebd1f 2024-11-12T10:54:17,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/964097de0e3d44579ea3fabd8a90663b is 50, key is test_row_0/C:col10/1731408855129/Put/seqid=0 2024-11-12T10:54:17,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741982_1158 (size=12151) 2024-11-12T10:54:17,566 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/964097de0e3d44579ea3fabd8a90663b 2024-11-12T10:54:17,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/b059d1e261ff48c590d8d2c7cff2b0da as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/b059d1e261ff48c590d8d2c7cff2b0da 2024-11-12T10:54:17,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:17,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33078 deadline: 1731408917567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:17,572 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in f3fbab0ede17db49ead8375d789c7d6a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:17,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:17,577 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/b059d1e261ff48c590d8d2c7cff2b0da, entries=150, sequenceid=210, filesize=30.4 K 2024-11-12T10:54:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731408917576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:17,578 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in f3fbab0ede17db49ead8375d789c7d6a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:17,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/3634c087ec4942d9b5dd4c64717ebd1f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/3634c087ec4942d9b5dd4c64717ebd1f 2024-11-12T10:54:17,584 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/3634c087ec4942d9b5dd4c64717ebd1f, entries=150, sequenceid=210, filesize=11.9 K 2024-11-12T10:54:17,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/964097de0e3d44579ea3fabd8a90663b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/964097de0e3d44579ea3fabd8a90663b 2024-11-12T10:54:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T10:54:17,591 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/964097de0e3d44579ea3fabd8a90663b, entries=150, sequenceid=210, filesize=11.9 K 2024-11-12T10:54:17,592 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for f3fbab0ede17db49ead8375d789c7d6a in 479ms, sequenceid=210, compaction requested=true 2024-11-12T10:54:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-12T10:54:17,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-12T10:54:17,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-12T10:54:17,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 633 msec 2024-11-12T10:54:17,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 638 msec 2024-11-12T10:54:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:17,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T10:54:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:17,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126cc7868c4d1b41f2afc1abddaa242516_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408857296/Put/seqid=0 2024-11-12T10:54:17,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741983_1159 (size=14794) 2024-11-12T10:54:17,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731408857645, max timestamp=1731408857660Num uniques 859; 2024-11-12T10:54:17,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731408857645, max timestamp=1731408857660Num uniques 859; 2024-11-12T10:54:17,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T10:54:17,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T10:54:17,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731408857645, max timestamp=1731408857660Num uniques 859; 2024-11-12T10:54:17,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T10:54:17,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408917714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:17,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408917714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:17,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408917716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:17,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408917827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:17,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408917827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408917827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:18,042 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:18,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:18,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408918037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408918037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:18,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408918037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:18,048 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126cc7868c4d1b41f2afc1abddaa242516_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126cc7868c4d1b41f2afc1abddaa242516_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:18,049 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/e8dfd351ce8a4d55a27f97547c47473d, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:18,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/e8dfd351ce8a4d55a27f97547c47473d is 175, key is test_row_0/A:col10/1731408857296/Put/seqid=0 2024-11-12T10:54:18,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741984_1160 (size=39749) 2024-11-12T10:54:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T10:54:18,098 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:18,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-12T10:54:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T10:54:18,100 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:18,101 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:18,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T10:54:18,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T10:54:18,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:18,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:18,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:18,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408918346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408918346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:18,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408918357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:18,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T10:54:18,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:18,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:18,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:18,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T10:54:18,455 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/e8dfd351ce8a4d55a27f97547c47473d 2024-11-12T10:54:18,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/6b510213ae3c48ef950dfeef5b3efee9 is 50, key is test_row_0/B:col10/1731408857296/Put/seqid=0 2024-11-12T10:54:18,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741985_1161 (size=12151) 2024-11-12T10:54:18,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T10:54:18,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:18,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:18,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:18,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T10:54:18,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:18,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:18,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:18,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T10:54:18,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408918857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:18,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408918857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:18,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T10:54:18,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:18,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:18,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:18,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:18,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408918867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:18,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/6b510213ae3c48ef950dfeef5b3efee9 2024-11-12T10:54:18,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:18,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:18,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/1c79216cae1245058b67f72ab07f1dcc is 50, key is test_row_0/C:col10/1731408857296/Put/seqid=0 2024-11-12T10:54:18,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741986_1162 (size=12151) 2024-11-12T10:54:19,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T10:54:19,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:19,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:19,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:19,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:19,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:19,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T10:54:19,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:19,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:19,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:19,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:19,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T10:54:19,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/1c79216cae1245058b67f72ab07f1dcc 2024-11-12T10:54:19,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/e8dfd351ce8a4d55a27f97547c47473d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/e8dfd351ce8a4d55a27f97547c47473d 2024-11-12T10:54:19,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/e8dfd351ce8a4d55a27f97547c47473d, entries=200, sequenceid=220, filesize=38.8 K 2024-11-12T10:54:19,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/6b510213ae3c48ef950dfeef5b3efee9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/6b510213ae3c48ef950dfeef5b3efee9 2024-11-12T10:54:19,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/6b510213ae3c48ef950dfeef5b3efee9, entries=150, sequenceid=220, filesize=11.9 K 2024-11-12T10:54:19,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/1c79216cae1245058b67f72ab07f1dcc as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/1c79216cae1245058b67f72ab07f1dcc 2024-11-12T10:54:19,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/1c79216cae1245058b67f72ab07f1dcc, entries=150, sequenceid=220, filesize=11.9 K 2024-11-12T10:54:19,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f3fbab0ede17db49ead8375d789c7d6a in 1681ms, sequenceid=220, compaction requested=true 2024-11-12T10:54:19,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:19,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:19,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:19,301 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-12T10:54:19,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:19,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:19,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:19,301 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-12T10:54:19,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:19,303 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164259 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-12T10:54:19,303 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60845 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-12T10:54:19,303 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/A is initiating minor compaction (all files) 2024-11-12T10:54:19,303 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/B is initiating minor compaction (all files) 2024-11-12T10:54:19,303 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/B in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:19,303 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/A in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:19,303 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/7ba992eba3344a46af01f32012bb6670, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/9889f7efbda84420a37d92f9070ee6d7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/344ffe34d0304aad89d579e40b846fc6, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/3634c087ec4942d9b5dd4c64717ebd1f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/6b510213ae3c48ef950dfeef5b3efee9] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=59.4 K 2024-11-12T10:54:19,303 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/c1518a51407148baa4ede524df352189, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/71acf022f4d748d18083038373a392b8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/eacbd119f0ed48c2a4d37289b26c17dd, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/b059d1e261ff48c590d8d2c7cff2b0da, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/e8dfd351ce8a4d55a27f97547c47473d] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=160.4 K 2024-11-12T10:54:19,303 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:19,303 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/c1518a51407148baa4ede524df352189, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/71acf022f4d748d18083038373a392b8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/eacbd119f0ed48c2a4d37289b26c17dd, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/b059d1e261ff48c590d8d2c7cff2b0da, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/e8dfd351ce8a4d55a27f97547c47473d] 2024-11-12T10:54:19,303 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ba992eba3344a46af01f32012bb6670, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731408852644 2024-11-12T10:54:19,304 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1518a51407148baa4ede524df352189, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731408852644 2024-11-12T10:54:19,304 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71acf022f4d748d18083038373a392b8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731408852752 2024-11-12T10:54:19,304 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 9889f7efbda84420a37d92f9070ee6d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731408852752 2024-11-12T10:54:19,304 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting eacbd119f0ed48c2a4d37289b26c17dd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731408853922 2024-11-12T10:54:19,304 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 344ffe34d0304aad89d579e40b846fc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731408853922 2024-11-12T10:54:19,304 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 3634c087ec4942d9b5dd4c64717ebd1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731408855117 2024-11-12T10:54:19,305 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting b059d1e261ff48c590d8d2c7cff2b0da, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731408855117 2024-11-12T10:54:19,305 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b510213ae3c48ef950dfeef5b3efee9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731408857291 2024-11-12T10:54:19,305 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8dfd351ce8a4d55a27f97547c47473d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731408857291 2024-11-12T10:54:19,315 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:19,317 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112feaea3b7bec8454e851d6977afe05586_f3fbab0ede17db49ead8375d789c7d6a store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:19,320 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#B#compaction#136 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:19,320 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112feaea3b7bec8454e851d6977afe05586_f3fbab0ede17db49ead8375d789c7d6a, store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:19,320 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/3badeac3d9c449e6a911519ad7f9ec0d is 50, key is test_row_0/B:col10/1731408857296/Put/seqid=0 2024-11-12T10:54:19,321 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112feaea3b7bec8454e851d6977afe05586_f3fbab0ede17db49ead8375d789c7d6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:19,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T10:54:19,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:19,324 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-12T10:54:19,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:19,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:19,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:19,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:19,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:19,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:19,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741987_1163 (size=12561) 2024-11-12T10:54:19,336 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/3badeac3d9c449e6a911519ad7f9ec0d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/3badeac3d9c449e6a911519ad7f9ec0d 2024-11-12T10:54:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122c0b7759244d46eab367554efaad9b23_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408857712/Put/seqid=0 2024-11-12T10:54:19,344 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/B of f3fbab0ede17db49ead8375d789c7d6a into 3badeac3d9c449e6a911519ad7f9ec0d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:19,344 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:19,344 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/B, priority=11, startTime=1731408859301; duration=0sec 2024-11-12T10:54:19,344 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:19,344 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:B 2024-11-12T10:54:19,344 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-12T10:54:19,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741988_1164 (size=4469) 2024-11-12T10:54:19,347 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60845 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-12T10:54:19,348 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/C is initiating minor compaction (all files) 2024-11-12T10:54:19,348 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/C in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:19,348 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/3af8c4591f224263b9235fbf0c93ff57, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ebc68785ed2940cd972266751f72690a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/2e6a10f10d934528a2270d460e88eb2d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/964097de0e3d44579ea3fabd8a90663b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/1c79216cae1245058b67f72ab07f1dcc] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=59.4 K 2024-11-12T10:54:19,348 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 3af8c4591f224263b9235fbf0c93ff57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731408852644 2024-11-12T10:54:19,349 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ebc68785ed2940cd972266751f72690a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731408852752 2024-11-12T10:54:19,349 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e6a10f10d934528a2270d460e88eb2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731408853922 2024-11-12T10:54:19,350 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 964097de0e3d44579ea3fabd8a90663b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731408855117 2024-11-12T10:54:19,350 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c79216cae1245058b67f72ab07f1dcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731408857291 2024-11-12T10:54:19,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741989_1165 (size=12304) 2024-11-12T10:54:19,361 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:19,361 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/1b68e158949d4062acd976589b98e227 is 50, key is test_row_0/C:col10/1731408857296/Put/seqid=0 2024-11-12T10:54:19,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741990_1166 (size=12561) 2024-11-12T10:54:19,378 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/1b68e158949d4062acd976589b98e227 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/1b68e158949d4062acd976589b98e227 2024-11-12T10:54:19,384 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/C of f3fbab0ede17db49ead8375d789c7d6a into 1b68e158949d4062acd976589b98e227(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:19,384 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:19,385 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/C, priority=11, startTime=1731408859301; duration=0sec 2024-11-12T10:54:19,385 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:19,385 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:C 2024-11-12T10:54:19,747 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#A#compaction#135 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:19,748 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/c408657629c54d3bb0fb011321708949 is 175, key is test_row_0/A:col10/1731408857296/Put/seqid=0 2024-11-12T10:54:19,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:19,755 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122c0b7759244d46eab367554efaad9b23_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122c0b7759244d46eab367554efaad9b23_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:19,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/79bacab0c2b34a2ba1a6ed89221344a4, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:19,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/79bacab0c2b34a2ba1a6ed89221344a4 is 175, key is test_row_0/A:col10/1731408857712/Put/seqid=0 2024-11-12T10:54:19,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741991_1167 (size=31515) 2024-11-12T10:54:19,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741992_1168 (size=31105) 2024-11-12T10:54:19,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:19,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408919890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:19,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408919890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:19,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408919891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:19,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408919997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:19,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408919997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:20,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408920007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:20,168 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/c408657629c54d3bb0fb011321708949 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/c408657629c54d3bb0fb011321708949 2024-11-12T10:54:20,171 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/79bacab0c2b34a2ba1a6ed89221344a4 2024-11-12T10:54:20,175 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/A of f3fbab0ede17db49ead8375d789c7d6a into c408657629c54d3bb0fb011321708949(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:20,175 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:20,175 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/A, priority=11, startTime=1731408859301; duration=0sec 2024-11-12T10:54:20,175 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:20,175 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:A 2024-11-12T10:54:20,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/fffc16d6e4cc4ebe92c9e0715029c60f is 50, key is test_row_0/B:col10/1731408857712/Put/seqid=0 2024-11-12T10:54:20,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741993_1169 (size=12151) 2024-11-12T10:54:20,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408920207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:20,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408920207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:20,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408920217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T10:54:20,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408920516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:20,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408920527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:20,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408920527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:20,586 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/fffc16d6e4cc4ebe92c9e0715029c60f 2024-11-12T10:54:20,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/c85d293220a6484d9f076cfd0343dc6f is 50, key is test_row_0/C:col10/1731408857712/Put/seqid=0 2024-11-12T10:54:20,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741994_1170 (size=12151) 2024-11-12T10:54:20,998 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/c85d293220a6484d9f076cfd0343dc6f 2024-11-12T10:54:21,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/79bacab0c2b34a2ba1a6ed89221344a4 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/79bacab0c2b34a2ba1a6ed89221344a4 2024-11-12T10:54:21,006 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/79bacab0c2b34a2ba1a6ed89221344a4, entries=150, sequenceid=255, filesize=30.4 K 2024-11-12T10:54:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/fffc16d6e4cc4ebe92c9e0715029c60f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/fffc16d6e4cc4ebe92c9e0715029c60f 2024-11-12T10:54:21,011 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/fffc16d6e4cc4ebe92c9e0715029c60f, entries=150, sequenceid=255, filesize=11.9 K 2024-11-12T10:54:21,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/c85d293220a6484d9f076cfd0343dc6f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/c85d293220a6484d9f076cfd0343dc6f 2024-11-12T10:54:21,016 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/c85d293220a6484d9f076cfd0343dc6f, entries=150, sequenceid=255, filesize=11.9 K 2024-11-12T10:54:21,017 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for f3fbab0ede17db49ead8375d789c7d6a in 1693ms, sequenceid=255, compaction requested=false 2024-11-12T10:54:21,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:21,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:21,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-12T10:54:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-12T10:54:21,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-12T10:54:21,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9180 sec 2024-11-12T10:54:21,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.9230 sec 2024-11-12T10:54:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:21,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T10:54:21,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:21,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:21,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:21,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:21,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:21,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:21,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126212d86c7bae42429183a913a9c5ce95_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408859889/Put/seqid=0 2024-11-12T10:54:21,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741995_1171 (size=14994) 2024-11-12T10:54:21,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408921059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:21,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408921060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:21,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408921063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:21,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408921167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:21,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408921167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:21,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:21,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408921177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:21,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408921377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:21,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408921377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:21,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408921387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:21,443 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:21,446 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126212d86c7bae42429183a913a9c5ce95_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126212d86c7bae42429183a913a9c5ce95_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:21,447 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/790abcd6f2fa4f1ea1f35a53e3e01c31, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:21,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/790abcd6f2fa4f1ea1f35a53e3e01c31 is 175, key is test_row_0/A:col10/1731408859889/Put/seqid=0 2024-11-12T10:54:21,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741996_1172 (size=39949) 2024-11-12T10:54:21,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408921687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:21,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408921687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:21,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408921697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:21,852 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/790abcd6f2fa4f1ea1f35a53e3e01c31 2024-11-12T10:54:21,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/e1d2771bf20046b0bf1c5b9c738c4726 is 50, key is test_row_0/B:col10/1731408859889/Put/seqid=0 2024-11-12T10:54:21,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741997_1173 (size=12301) 2024-11-12T10:54:22,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408922197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:22,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408922197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:22,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408922206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T10:54:22,267 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:22,268 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-12T10:54:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T10:54:22,271 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:22,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/e1d2771bf20046b0bf1c5b9c738c4726 2024-11-12T10:54:22,272 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:22,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:22,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ebb0987570964c99861a319885481b6a is 50, key is test_row_0/C:col10/1731408859889/Put/seqid=0 2024-11-12T10:54:22,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741998_1174 (size=12301) 2024-11-12T10:54:22,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ebb0987570964c99861a319885481b6a 2024-11-12T10:54:22,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/790abcd6f2fa4f1ea1f35a53e3e01c31 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/790abcd6f2fa4f1ea1f35a53e3e01c31 2024-11-12T10:54:22,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/790abcd6f2fa4f1ea1f35a53e3e01c31, entries=200, sequenceid=270, filesize=39.0 K 2024-11-12T10:54:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/e1d2771bf20046b0bf1c5b9c738c4726 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/e1d2771bf20046b0bf1c5b9c738c4726 2024-11-12T10:54:22,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/e1d2771bf20046b0bf1c5b9c738c4726, entries=150, sequenceid=270, filesize=12.0 K 2024-11-12T10:54:22,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ebb0987570964c99861a319885481b6a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ebb0987570964c99861a319885481b6a 2024-11-12T10:54:22,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ebb0987570964c99861a319885481b6a, entries=150, sequenceid=270, filesize=12.0 K 2024-11-12T10:54:22,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f3fbab0ede17db49ead8375d789c7d6a in 1290ms, sequenceid=270, compaction requested=true 2024-11-12T10:54:22,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:22,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:22,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:22,319 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:22,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:22,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:22,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:22,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T10:54:22,319 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:22,320 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102569 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:22,320 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/A is initiating minor compaction (all files) 2024-11-12T10:54:22,321 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/A in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:22,321 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/c408657629c54d3bb0fb011321708949, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/79bacab0c2b34a2ba1a6ed89221344a4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/790abcd6f2fa4f1ea1f35a53e3e01c31] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=100.2 K 2024-11-12T10:54:22,321 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:22,321 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/B is initiating minor compaction (all files) 2024-11-12T10:54:22,321 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:22,321 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/B in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:22,321 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/c408657629c54d3bb0fb011321708949, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/79bacab0c2b34a2ba1a6ed89221344a4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/790abcd6f2fa4f1ea1f35a53e3e01c31] 2024-11-12T10:54:22,321 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/3badeac3d9c449e6a911519ad7f9ec0d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/fffc16d6e4cc4ebe92c9e0715029c60f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/e1d2771bf20046b0bf1c5b9c738c4726] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=36.1 K 2024-11-12T10:54:22,321 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting c408657629c54d3bb0fb011321708949, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731408857291 2024-11-12T10:54:22,322 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79bacab0c2b34a2ba1a6ed89221344a4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731408857711 2024-11-12T10:54:22,322 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 3badeac3d9c449e6a911519ad7f9ec0d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731408857291 2024-11-12T10:54:22,322 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting fffc16d6e4cc4ebe92c9e0715029c60f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731408857711 2024-11-12T10:54:22,323 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 790abcd6f2fa4f1ea1f35a53e3e01c31, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731408859880 2024-11-12T10:54:22,323 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting e1d2771bf20046b0bf1c5b9c738c4726, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731408859880 2024-11-12T10:54:22,329 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:22,335 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#B#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:22,335 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411128efb0c256b7445bc8460c573f29bb446_f3fbab0ede17db49ead8375d789c7d6a store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:22,335 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/9bb31d43613c4d769cb9416e243471b1 is 50, key is test_row_0/B:col10/1731408859889/Put/seqid=0 2024-11-12T10:54:22,337 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411128efb0c256b7445bc8460c573f29bb446_f3fbab0ede17db49ead8375d789c7d6a, store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:22,337 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128efb0c256b7445bc8460c573f29bb446_f3fbab0ede17db49ead8375d789c7d6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:22,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741999_1175 (size=12813) 2024-11-12T10:54:22,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742000_1176 (size=4469) 2024-11-12T10:54:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T10:54:22,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T10:54:22,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:22,425 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T10:54:22,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:22,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:22,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:22,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:22,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:22,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:22,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122b2ba5406b8d47e1b2f1a42360648706_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408861053/Put/seqid=0 2024-11-12T10:54:22,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742001_1177 (size=12454) 2024-11-12T10:54:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T10:54:22,745 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/9bb31d43613c4d769cb9416e243471b1 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/9bb31d43613c4d769cb9416e243471b1 2024-11-12T10:54:22,750 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#A#compaction#144 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:22,750 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/66117602eae8435ba201d6dd33c8f307 is 175, key is test_row_0/A:col10/1731408859889/Put/seqid=0 2024-11-12T10:54:22,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742002_1178 (size=31767) 2024-11-12T10:54:22,759 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/B of f3fbab0ede17db49ead8375d789c7d6a into 9bb31d43613c4d769cb9416e243471b1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:22,760 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:22,760 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/B, priority=13, startTime=1731408862319; duration=0sec 2024-11-12T10:54:22,760 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:22,760 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:B 2024-11-12T10:54:22,760 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:22,764 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:22,765 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/C is initiating minor compaction (all files) 2024-11-12T10:54:22,765 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/C in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:22,765 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/1b68e158949d4062acd976589b98e227, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/c85d293220a6484d9f076cfd0343dc6f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ebb0987570964c99861a319885481b6a] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=36.1 K 2024-11-12T10:54:22,766 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b68e158949d4062acd976589b98e227, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731408857291 2024-11-12T10:54:22,766 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting c85d293220a6484d9f076cfd0343dc6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731408857711 2024-11-12T10:54:22,768 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ebb0987570964c99861a319885481b6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731408859880 2024-11-12T10:54:22,768 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/66117602eae8435ba201d6dd33c8f307 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/66117602eae8435ba201d6dd33c8f307 2024-11-12T10:54:22,775 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/A of f3fbab0ede17db49ead8375d789c7d6a into 66117602eae8435ba201d6dd33c8f307(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:22,775 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:22,775 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/A, priority=13, startTime=1731408862319; duration=0sec 2024-11-12T10:54:22,775 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:22,775 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:A 2024-11-12T10:54:22,779 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:22,779 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/f75d90e7dd53408bbfef4ae8838409a6 is 50, key is test_row_0/C:col10/1731408859889/Put/seqid=0 2024-11-12T10:54:22,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742003_1179 (size=12813) 2024-11-12T10:54:22,795 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/f75d90e7dd53408bbfef4ae8838409a6 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/f75d90e7dd53408bbfef4ae8838409a6 2024-11-12T10:54:22,805 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/C of f3fbab0ede17db49ead8375d789c7d6a into f75d90e7dd53408bbfef4ae8838409a6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:22,805 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:22,805 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/C, priority=13, startTime=1731408862319; duration=0sec 2024-11-12T10:54:22,805 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:22,805 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:C 2024-11-12T10:54:22,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:22,840 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122b2ba5406b8d47e1b2f1a42360648706_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122b2ba5406b8d47e1b2f1a42360648706_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:22,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/ce2a467f1b144f87a69e3b24c6a957d7, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:22,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/ce2a467f1b144f87a69e3b24c6a957d7 is 175, key is test_row_0/A:col10/1731408861053/Put/seqid=0 2024-11-12T10:54:22,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742004_1180 (size=31255) 2024-11-12T10:54:22,847 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/ce2a467f1b144f87a69e3b24c6a957d7 2024-11-12T10:54:22,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/b69d79f31d0942a28af5c3ad836c2315 is 50, key is test_row_0/B:col10/1731408861053/Put/seqid=0 2024-11-12T10:54:22,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742005_1181 (size=12301) 2024-11-12T10:54:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T10:54:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:23,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:23,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408923218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:23,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:23,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408923223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:23,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:23,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:23,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408923223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:23,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:23,265 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/b69d79f31d0942a28af5c3ad836c2315 2024-11-12T10:54:23,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ddd2b216db954e8e92c9ef39094d166f is 50, key is test_row_0/C:col10/1731408861053/Put/seqid=0 2024-11-12T10:54:23,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742006_1182 (size=12301) 2024-11-12T10:54:23,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408923327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408923327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408923327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T10:54:23,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408923537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:23,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408923537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:23,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408923537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:23,680 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ddd2b216db954e8e92c9ef39094d166f 2024-11-12T10:54:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/ce2a467f1b144f87a69e3b24c6a957d7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/ce2a467f1b144f87a69e3b24c6a957d7 2024-11-12T10:54:23,689 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/ce2a467f1b144f87a69e3b24c6a957d7, entries=150, sequenceid=290, filesize=30.5 K 2024-11-12T10:54:23,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/b69d79f31d0942a28af5c3ad836c2315 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/b69d79f31d0942a28af5c3ad836c2315 2024-11-12T10:54:23,695 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/b69d79f31d0942a28af5c3ad836c2315, entries=150, sequenceid=290, filesize=12.0 K 2024-11-12T10:54:23,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ddd2b216db954e8e92c9ef39094d166f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ddd2b216db954e8e92c9ef39094d166f 2024-11-12T10:54:23,700 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ddd2b216db954e8e92c9ef39094d166f, entries=150, sequenceid=290, filesize=12.0 K 2024-11-12T10:54:23,701 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for f3fbab0ede17db49ead8375d789c7d6a in 1276ms, sequenceid=290, compaction requested=false 2024-11-12T10:54:23,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:23,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:23,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-12T10:54:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-12T10:54:23,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-12T10:54:23,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4310 sec 2024-11-12T10:54:23,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4370 sec 2024-11-12T10:54:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:23,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T10:54:23,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:23,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:23,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:23,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:23,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:23,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:23,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111286099348d0a84f5da15e8aa790b52e4f_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408863847/Put/seqid=0 2024-11-12T10:54:23,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742007_1183 (size=14994) 2024-11-12T10:54:23,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408923898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:23,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408923900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:23,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:23,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408923901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:24,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408924007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:24,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408924007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:24,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408924017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:24,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:24,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408924217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408924217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:24,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408924227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:24,270 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:24,274 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111286099348d0a84f5da15e8aa790b52e4f_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111286099348d0a84f5da15e8aa790b52e4f_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:24,275 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/d6cd412abd3f4f6a99db63dde1a63bca, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:24,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/d6cd412abd3f4f6a99db63dde1a63bca is 175, key is test_row_0/A:col10/1731408863847/Put/seqid=0 2024-11-12T10:54:24,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742008_1184 (size=39949) 2024-11-12T10:54:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T10:54:24,427 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:24,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-12T10:54:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T10:54:24,429 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:24,430 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:24,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:24,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:24,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408924527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408924527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:24,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:24,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T10:54:24,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408924537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:24,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T10:54:24,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:24,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:24,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:24,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:24,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:24,680 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/d6cd412abd3f4f6a99db63dde1a63bca 2024-11-12T10:54:24,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/79e5e1326a6d45909342e6181f4fe98f is 50, key is test_row_0/B:col10/1731408863847/Put/seqid=0 2024-11-12T10:54:24,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742009_1185 (size=12301) 2024-11-12T10:54:24,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T10:54:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:24,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T10:54:24,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T10:54:24,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:24,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:24,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:24,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:24,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:25,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408925037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:25,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408925037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:25,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T10:54:25,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:25,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:25,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:25,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408925047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:25,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T10:54:25,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/79e5e1326a6d45909342e6181f4fe98f 2024-11-12T10:54:25,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/8afb18f902914bd4bf66431c30a6fef0 is 50, key is test_row_0/C:col10/1731408863847/Put/seqid=0 2024-11-12T10:54:25,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742010_1186 (size=12301) 2024-11-12T10:54:25,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T10:54:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:25,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:25,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T10:54:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:25,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:25,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T10:54:25,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:25,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:25,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:25,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:25,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:25,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/8afb18f902914bd4bf66431c30a6fef0 2024-11-12T10:54:25,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/d6cd412abd3f4f6a99db63dde1a63bca as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/d6cd412abd3f4f6a99db63dde1a63bca 2024-11-12T10:54:25,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/d6cd412abd3f4f6a99db63dde1a63bca, entries=200, sequenceid=307, filesize=39.0 K 2024-11-12T10:54:25,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/79e5e1326a6d45909342e6181f4fe98f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/79e5e1326a6d45909342e6181f4fe98f 2024-11-12T10:54:25,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/79e5e1326a6d45909342e6181f4fe98f, entries=150, sequenceid=307, filesize=12.0 K 2024-11-12T10:54:25,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/8afb18f902914bd4bf66431c30a6fef0 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/8afb18f902914bd4bf66431c30a6fef0 2024-11-12T10:54:25,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/8afb18f902914bd4bf66431c30a6fef0, entries=150, sequenceid=307, filesize=12.0 K 2024-11-12T10:54:25,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f3fbab0ede17db49ead8375d789c7d6a in 1678ms, sequenceid=307, compaction requested=true 2024-11-12T10:54:25,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:25,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:25,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:25,526 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:25,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:25,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:25,526 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:25,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:25,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:25,527 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:25,527 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:25,527 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/A is initiating minor compaction (all files) 2024-11-12T10:54:25,527 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/B is initiating minor compaction (all files) 2024-11-12T10:54:25,527 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/B in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:25,527 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/A in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:25,527 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/9bb31d43613c4d769cb9416e243471b1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/b69d79f31d0942a28af5c3ad836c2315, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/79e5e1326a6d45909342e6181f4fe98f] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=36.5 K 2024-11-12T10:54:25,527 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/66117602eae8435ba201d6dd33c8f307, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/ce2a467f1b144f87a69e3b24c6a957d7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/d6cd412abd3f4f6a99db63dde1a63bca] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=100.6 K 2024-11-12T10:54:25,527 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:25,528 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/66117602eae8435ba201d6dd33c8f307, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/ce2a467f1b144f87a69e3b24c6a957d7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/d6cd412abd3f4f6a99db63dde1a63bca] 2024-11-12T10:54:25,529 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bb31d43613c4d769cb9416e243471b1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731408859880 2024-11-12T10:54:25,529 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66117602eae8435ba201d6dd33c8f307, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731408859880 2024-11-12T10:54:25,529 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting b69d79f31d0942a28af5c3ad836c2315, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731408861053 2024-11-12T10:54:25,529 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce2a467f1b144f87a69e3b24c6a957d7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731408861053 2024-11-12T10:54:25,529 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 79e5e1326a6d45909342e6181f4fe98f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731408863215 2024-11-12T10:54:25,529 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6cd412abd3f4f6a99db63dde1a63bca, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731408863215 2024-11-12T10:54:25,550 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:25,551 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#B#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:25,552 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/1f6263bf855a408781d92743715174b7 is 50, key is test_row_0/B:col10/1731408863847/Put/seqid=0 2024-11-12T10:54:25,552 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112cdd592fc44534c3da5bd30e835dfb146_f3fbab0ede17db49ead8375d789c7d6a store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:25,554 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112cdd592fc44534c3da5bd30e835dfb146_f3fbab0ede17db49ead8375d789c7d6a, store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:25,554 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112cdd592fc44534c3da5bd30e835dfb146_f3fbab0ede17db49ead8375d789c7d6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:25,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742011_1187 (size=12915) 2024-11-12T10:54:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T10:54:25,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742012_1188 (size=4469) 2024-11-12T10:54:25,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T10:54:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:25,652 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T10:54:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:25,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111271f427c532fa4748b5699e357430f8ff_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408863899/Put/seqid=0 2024-11-12T10:54:25,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742013_1189 (size=12454) 2024-11-12T10:54:25,963 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/1f6263bf855a408781d92743715174b7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/1f6263bf855a408781d92743715174b7 2024-11-12T10:54:25,968 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/B of f3fbab0ede17db49ead8375d789c7d6a into 1f6263bf855a408781d92743715174b7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:25,968 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:25,968 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/B, priority=13, startTime=1731408865526; duration=0sec 2024-11-12T10:54:25,968 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:25,968 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:B 2024-11-12T10:54:25,968 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:25,970 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:25,970 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#A#compaction#154 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:25,970 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): f3fbab0ede17db49ead8375d789c7d6a/C is initiating minor compaction (all files) 2024-11-12T10:54:25,970 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3fbab0ede17db49ead8375d789c7d6a/C in TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:25,970 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/f75d90e7dd53408bbfef4ae8838409a6, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ddd2b216db954e8e92c9ef39094d166f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/8afb18f902914bd4bf66431c30a6fef0] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp, totalSize=36.5 K 2024-11-12T10:54:25,970 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/5d2fa5f6a69643f694b20b5cb2d39ab7 is 175, key is test_row_0/A:col10/1731408863847/Put/seqid=0 2024-11-12T10:54:25,970 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting f75d90e7dd53408bbfef4ae8838409a6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731408859880 2024-11-12T10:54:25,971 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ddd2b216db954e8e92c9ef39094d166f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731408861053 2024-11-12T10:54:25,971 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 8afb18f902914bd4bf66431c30a6fef0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731408863215 2024-11-12T10:54:25,979 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3fbab0ede17db49ead8375d789c7d6a#C#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:25,980 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/f8b56ec6b65f490491204faa9b7d2af7 is 50, key is test_row_0/C:col10/1731408863847/Put/seqid=0 2024-11-12T10:54:25,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742014_1190 (size=31869) 2024-11-12T10:54:25,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742015_1191 (size=12915) 2024-11-12T10:54:25,994 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/5d2fa5f6a69643f694b20b5cb2d39ab7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/5d2fa5f6a69643f694b20b5cb2d39ab7 2024-11-12T10:54:26,002 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/A of f3fbab0ede17db49ead8375d789c7d6a into 5d2fa5f6a69643f694b20b5cb2d39ab7(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:26,002 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:26,002 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/f8b56ec6b65f490491204faa9b7d2af7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/f8b56ec6b65f490491204faa9b7d2af7 2024-11-12T10:54:26,002 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/A, priority=13, startTime=1731408865526; duration=0sec 2024-11-12T10:54:26,002 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:26,002 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:A 2024-11-12T10:54:26,009 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3fbab0ede17db49ead8375d789c7d6a/C of f3fbab0ede17db49ead8375d789c7d6a into f8b56ec6b65f490491204faa9b7d2af7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:26,009 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:26,009 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., storeName=f3fbab0ede17db49ead8375d789c7d6a/C, priority=13, startTime=1731408865526; duration=0sec 2024-11-12T10:54:26,009 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:26,009 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:C 2024-11-12T10:54:26,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. as already flushing 2024-11-12T10:54:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:26,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:26,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408926063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:26,067 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111271f427c532fa4748b5699e357430f8ff_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111271f427c532fa4748b5699e357430f8ff_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:26,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/1c68cc5a16924039a0f19240a0c3528f, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:26,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/1c68cc5a16924039a0f19240a0c3528f is 175, key is test_row_0/A:col10/1731408863899/Put/seqid=0 2024-11-12T10:54:26,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408926065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:26,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:26,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742016_1192 (size=31255) 2024-11-12T10:54:26,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33088 deadline: 1731408926067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:26,073 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in f3fbab0ede17db49ead8375d789c7d6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:26,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:26,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408926177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408926177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:26,235 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-11-12T10:54:26,235 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:26,235 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:26,235 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:26,235 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:26,235 INFO [Thread-605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-605. 2024-11-12T10:54:26,235 DEBUG [Thread-605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:26,235 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:26,235 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:26,235 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:26,237 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-11-12T10:54:26,237 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:26,237 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:26,237 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:26,237 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:26,238 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-11-12T10:54:26,238 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:26,238 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:26,238 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:26,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:26,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:26,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408926387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408926387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:26,474 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/1c68cc5a16924039a0f19240a0c3528f 2024-11-12T10:54:26,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/0c9dca3387384ba7a9efecba50430c1b is 50, key is test_row_0/B:col10/1731408863899/Put/seqid=0 2024-11-12T10:54:26,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742017_1193 (size=12301) 2024-11-12T10:54:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T10:54:26,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:26,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408926697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408926697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:26,894 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/0c9dca3387384ba7a9efecba50430c1b 2024-11-12T10:54:26,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/14b7c439ed604adba2058ff236f4ce61 is 50, key is test_row_0/C:col10/1731408863899/Put/seqid=0 2024-11-12T10:54:26,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742018_1194 (size=12301) 2024-11-12T10:54:27,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:27,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33152 deadline: 1731408927207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33138 deadline: 1731408927207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:54:27,312 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/14b7c439ed604adba2058ff236f4ce61 2024-11-12T10:54:27,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/1c68cc5a16924039a0f19240a0c3528f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/1c68cc5a16924039a0f19240a0c3528f 2024-11-12T10:54:27,329 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/1c68cc5a16924039a0f19240a0c3528f, entries=150, sequenceid=325, filesize=30.5 K 2024-11-12T10:54:27,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/0c9dca3387384ba7a9efecba50430c1b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/0c9dca3387384ba7a9efecba50430c1b 2024-11-12T10:54:27,337 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/0c9dca3387384ba7a9efecba50430c1b, entries=150, sequenceid=325, filesize=12.0 K 2024-11-12T10:54:27,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/14b7c439ed604adba2058ff236f4ce61 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/14b7c439ed604adba2058ff236f4ce61 2024-11-12T10:54:27,344 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/14b7c439ed604adba2058ff236f4ce61, entries=150, sequenceid=325, filesize=12.0 K 2024-11-12T10:54:27,345 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f3fbab0ede17db49ead8375d789c7d6a in 1693ms, sequenceid=325, compaction requested=false 2024-11-12T10:54:27,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:27,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:27,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-12T10:54:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-12T10:54:27,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-12T10:54:27,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9170 sec 2024-11-12T10:54:27,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.9230 sec 2024-11-12T10:54:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:27,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T10:54:27,631 INFO [Thread-596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-596. 2024-11-12T10:54:27,631 DEBUG [Thread-596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:27,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:27,631 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:27,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:27,631 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:27,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:27,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:27,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:27,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:27,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:27,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e708afbb5d4746d2a1411b3d7f2bb617_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408866061/Put/seqid=0 2024-11-12T10:54:27,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742019_1195 (size=12454) 2024-11-12T10:54:27,678 INFO [Thread-588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-588. 2024-11-12T10:54:27,678 DEBUG [Thread-588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:27,678 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:27,678 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:27,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:28,048 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:28,053 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e708afbb5d4746d2a1411b3d7f2bb617_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e708afbb5d4746d2a1411b3d7f2bb617_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:28,055 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/4450bc82f5d04defafc6fc765145039b, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:28,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/4450bc82f5d04defafc6fc765145039b is 175, key is test_row_0/A:col10/1731408866061/Put/seqid=0 2024-11-12T10:54:28,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742020_1196 (size=31255) 2024-11-12T10:54:28,090 INFO [Thread-594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-594. 2024-11-12T10:54:28,090 DEBUG [Thread-594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:28,090 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:28,090 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:28,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:28,220 INFO [Thread-592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-592. 2024-11-12T10:54:28,220 INFO [Thread-590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-590. 2024-11-12T10:54:28,220 DEBUG [Thread-592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:28,220 DEBUG [Thread-590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:28,220 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:28,220 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:28,221 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:28,221 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:28,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:28,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:28,462 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/4450bc82f5d04defafc6fc765145039b 2024-11-12T10:54:28,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/1d03a0902d0347ad92efecc1d09da7d1 is 50, key is test_row_0/B:col10/1731408866061/Put/seqid=0 2024-11-12T10:54:28,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742021_1197 (size=12301) 2024-11-12T10:54:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T10:54:28,598 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:28,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T10:54:28,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-12T10:54:28,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-12T10:54:28,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-12T10:54:28,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-12T10:54:28,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-12T10:54:28,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T10:54:28,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3245 2024-11-12T10:54:28,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3329 2024-11-12T10:54:28,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T10:54:28,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1384 2024-11-12T10:54:28,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4152 rows 2024-11-12T10:54:28,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1381 2024-11-12T10:54:28,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4140 rows 2024-11-12T10:54:28,600 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T10:54:28,600 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T10:54:28,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:28,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:28,601 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:28,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T10:54:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T10:54:28,608 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408868607"}]},"ts":"1731408868607"} 2024-11-12T10:54:28,610 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T10:54:28,610 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T10:54:28,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T10:54:28,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, UNASSIGN}] 2024-11-12T10:54:28,614 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, UNASSIGN 2024-11-12T10:54:28,615 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=f3fbab0ede17db49ead8375d789c7d6a, regionState=CLOSING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:54:28,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, UNASSIGN because future has completed 2024-11-12T10:54:28,617 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T10:54:28,617 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:54:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T10:54:28,770 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:28,770 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T10:54:28,770 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing f3fbab0ede17db49ead8375d789c7d6a, disabling compactions & flushes 2024-11-12T10:54:28,770 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:28,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/1d03a0902d0347ad92efecc1d09da7d1 2024-11-12T10:54:28,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/0d58a53f4fa14598bda72a0d41aaa1d6 is 50, key is test_row_0/C:col10/1731408866061/Put/seqid=0 2024-11-12T10:54:28,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742022_1198 (size=12301) 2024-11-12T10:54:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T10:54:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T10:54:29,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/0d58a53f4fa14598bda72a0d41aaa1d6 2024-11-12T10:54:29,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/4450bc82f5d04defafc6fc765145039b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/4450bc82f5d04defafc6fc765145039b 2024-11-12T10:54:29,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/4450bc82f5d04defafc6fc765145039b, entries=150, sequenceid=343, filesize=30.5 K 2024-11-12T10:54:29,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/1d03a0902d0347ad92efecc1d09da7d1 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/1d03a0902d0347ad92efecc1d09da7d1 2024-11-12T10:54:29,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/1d03a0902d0347ad92efecc1d09da7d1, entries=150, sequenceid=343, filesize=12.0 K 2024-11-12T10:54:29,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/0d58a53f4fa14598bda72a0d41aaa1d6 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/0d58a53f4fa14598bda72a0d41aaa1d6 2024-11-12T10:54:29,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/0d58a53f4fa14598bda72a0d41aaa1d6, entries=150, sequenceid=343, filesize=12.0 K 2024-11-12T10:54:29,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=26.84 KB/27480 for f3fbab0ede17db49ead8375d789c7d6a in 1700ms, sequenceid=343, compaction requested=true 2024-11-12T10:54:29,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3fbab0ede17db49ead8375d789c7d6a: 2024-11-12T10:54:29,330 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:29,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:29,330 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:29,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:29,330 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. after waiting 0 ms 2024-11-12T10:54:29,330 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:29,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:29,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:29,330 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. because compaction request was cancelled 2024-11-12T10:54:29,330 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing f3fbab0ede17db49ead8375d789c7d6a 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T10:54:29,330 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:A 2024-11-12T10:54:29,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3fbab0ede17db49ead8375d789c7d6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:29,330 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. because compaction request was cancelled 2024-11-12T10:54:29,330 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. because compaction request was cancelled 2024-11-12T10:54:29,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:29,330 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:C 2024-11-12T10:54:29,330 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3fbab0ede17db49ead8375d789c7d6a:B 2024-11-12T10:54:29,330 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=A 2024-11-12T10:54:29,330 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:29,331 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=B 2024-11-12T10:54:29,331 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:29,331 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3fbab0ede17db49ead8375d789c7d6a, store=C 2024-11-12T10:54:29,331 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:29,337 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112925d13bcce804278a08e4f38b908750a_f3fbab0ede17db49ead8375d789c7d6a is 50, key is test_row_0/A:col10/1731408867668/Put/seqid=0 2024-11-12T10:54:29,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742023_1199 (size=12454) 2024-11-12T10:54:29,741 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:29,748 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112925d13bcce804278a08e4f38b908750a_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112925d13bcce804278a08e4f38b908750a_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T10:54:29,749 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/ea5c541c786d44b8b1e074b1de760e59, store: [table=TestAcidGuarantees family=A region=f3fbab0ede17db49ead8375d789c7d6a] 2024-11-12T10:54:29,750 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/ea5c541c786d44b8b1e074b1de760e59 is 175, key is test_row_0/A:col10/1731408867668/Put/seqid=0 2024-11-12T10:54:29,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742024_1200 (size=31255) 2024-11-12T10:54:30,157 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/ea5c541c786d44b8b1e074b1de760e59 2024-11-12T10:54:30,166 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/54c42989cf8a4db78f4d1cc80bb73adc is 50, key is test_row_0/B:col10/1731408867668/Put/seqid=0 2024-11-12T10:54:30,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742025_1201 (size=12301) 2024-11-12T10:54:30,573 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/54c42989cf8a4db78f4d1cc80bb73adc 2024-11-12T10:54:30,585 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ce6cf7dd6d7e4f8ba6ebf467a5a2e0db is 50, key is test_row_0/C:col10/1731408867668/Put/seqid=0 2024-11-12T10:54:30,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742026_1202 (size=12301) 2024-11-12T10:54:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T10:54:30,990 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ce6cf7dd6d7e4f8ba6ebf467a5a2e0db 2024-11-12T10:54:30,995 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/A/ea5c541c786d44b8b1e074b1de760e59 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/ea5c541c786d44b8b1e074b1de760e59 2024-11-12T10:54:30,999 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/ea5c541c786d44b8b1e074b1de760e59, entries=150, sequenceid=350, filesize=30.5 K 2024-11-12T10:54:31,000 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/B/54c42989cf8a4db78f4d1cc80bb73adc as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/54c42989cf8a4db78f4d1cc80bb73adc 2024-11-12T10:54:31,004 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/54c42989cf8a4db78f4d1cc80bb73adc, entries=150, sequenceid=350, filesize=12.0 K 2024-11-12T10:54:31,005 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/.tmp/C/ce6cf7dd6d7e4f8ba6ebf467a5a2e0db as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ce6cf7dd6d7e4f8ba6ebf467a5a2e0db 2024-11-12T10:54:31,010 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ce6cf7dd6d7e4f8ba6ebf467a5a2e0db, entries=150, sequenceid=350, filesize=12.0 K 2024-11-12T10:54:31,011 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for f3fbab0ede17db49ead8375d789c7d6a in 1681ms, sequenceid=350, compaction requested=true 2024-11-12T10:54:31,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/1813b698abac442db5a79dfce2f6a969, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/53bc9a10402941ec859fccbd0762a5a5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/f894226f031144158a40c6655b40966f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/b036afe4c3e24b11a3e30071d0d31261, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/afda5678fff1499a8f923d37736d1904, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/890be27b4aa047559653d99f9dd3df4e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/037fba659c00423fab261a762e390d07, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/c1518a51407148baa4ede524df352189, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/71acf022f4d748d18083038373a392b8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/eacbd119f0ed48c2a4d37289b26c17dd, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/b059d1e261ff48c590d8d2c7cff2b0da, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/e8dfd351ce8a4d55a27f97547c47473d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/c408657629c54d3bb0fb011321708949, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/79bacab0c2b34a2ba1a6ed89221344a4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/790abcd6f2fa4f1ea1f35a53e3e01c31, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/66117602eae8435ba201d6dd33c8f307, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/ce2a467f1b144f87a69e3b24c6a957d7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/d6cd412abd3f4f6a99db63dde1a63bca] to archive 2024-11-12T10:54:31,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:54:31,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/1813b698abac442db5a79dfce2f6a969 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/1813b698abac442db5a79dfce2f6a969 2024-11-12T10:54:31,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/53bc9a10402941ec859fccbd0762a5a5 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/53bc9a10402941ec859fccbd0762a5a5 2024-11-12T10:54:31,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/f894226f031144158a40c6655b40966f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/f894226f031144158a40c6655b40966f 2024-11-12T10:54:31,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/b036afe4c3e24b11a3e30071d0d31261 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/b036afe4c3e24b11a3e30071d0d31261 2024-11-12T10:54:31,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/afda5678fff1499a8f923d37736d1904 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/afda5678fff1499a8f923d37736d1904 2024-11-12T10:54:31,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/890be27b4aa047559653d99f9dd3df4e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/890be27b4aa047559653d99f9dd3df4e 2024-11-12T10:54:31,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/037fba659c00423fab261a762e390d07 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/037fba659c00423fab261a762e390d07 2024-11-12T10:54:31,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/c1518a51407148baa4ede524df352189 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/c1518a51407148baa4ede524df352189 2024-11-12T10:54:31,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/71acf022f4d748d18083038373a392b8 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/71acf022f4d748d18083038373a392b8 2024-11-12T10:54:31,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/eacbd119f0ed48c2a4d37289b26c17dd to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/eacbd119f0ed48c2a4d37289b26c17dd 2024-11-12T10:54:31,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/b059d1e261ff48c590d8d2c7cff2b0da to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/b059d1e261ff48c590d8d2c7cff2b0da 2024-11-12T10:54:31,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/e8dfd351ce8a4d55a27f97547c47473d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/e8dfd351ce8a4d55a27f97547c47473d 2024-11-12T10:54:31,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/c408657629c54d3bb0fb011321708949 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/c408657629c54d3bb0fb011321708949 2024-11-12T10:54:31,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/79bacab0c2b34a2ba1a6ed89221344a4 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/79bacab0c2b34a2ba1a6ed89221344a4 2024-11-12T10:54:31,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/790abcd6f2fa4f1ea1f35a53e3e01c31 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/790abcd6f2fa4f1ea1f35a53e3e01c31 2024-11-12T10:54:31,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/66117602eae8435ba201d6dd33c8f307 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/66117602eae8435ba201d6dd33c8f307 2024-11-12T10:54:31,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/ce2a467f1b144f87a69e3b24c6a957d7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/ce2a467f1b144f87a69e3b24c6a957d7 2024-11-12T10:54:31,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/d6cd412abd3f4f6a99db63dde1a63bca to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/d6cd412abd3f4f6a99db63dde1a63bca 2024-11-12T10:54:31,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/ebfa6f0ccaba461e8d3d1a03d72b0c36, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/1d13a1fb26554814946442cd97eb7f13, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/e7478eba89524deeb03e5a1454e94eb5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/40e2d961b1c1432d90faeb7ce5fd22ff, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/b0369996b6fe4b3e91a0179bc6961512, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/adcd16868df446ed81f9e1799e45517a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/7ba992eba3344a46af01f32012bb6670, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/284aaea047974f758b7d2a108b43cef7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/9889f7efbda84420a37d92f9070ee6d7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/344ffe34d0304aad89d579e40b846fc6, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/3634c087ec4942d9b5dd4c64717ebd1f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/3badeac3d9c449e6a911519ad7f9ec0d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/6b510213ae3c48ef950dfeef5b3efee9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/fffc16d6e4cc4ebe92c9e0715029c60f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/9bb31d43613c4d769cb9416e243471b1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/e1d2771bf20046b0bf1c5b9c738c4726, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/b69d79f31d0942a28af5c3ad836c2315, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/79e5e1326a6d45909342e6181f4fe98f] to archive 2024-11-12T10:54:31,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:54:31,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/ebfa6f0ccaba461e8d3d1a03d72b0c36 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/ebfa6f0ccaba461e8d3d1a03d72b0c36 2024-11-12T10:54:31,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/1d13a1fb26554814946442cd97eb7f13 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/1d13a1fb26554814946442cd97eb7f13 2024-11-12T10:54:31,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/e7478eba89524deeb03e5a1454e94eb5 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/e7478eba89524deeb03e5a1454e94eb5 2024-11-12T10:54:31,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/40e2d961b1c1432d90faeb7ce5fd22ff to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/40e2d961b1c1432d90faeb7ce5fd22ff 2024-11-12T10:54:31,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/b0369996b6fe4b3e91a0179bc6961512 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/b0369996b6fe4b3e91a0179bc6961512 2024-11-12T10:54:31,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/adcd16868df446ed81f9e1799e45517a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/adcd16868df446ed81f9e1799e45517a 2024-11-12T10:54:31,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/7ba992eba3344a46af01f32012bb6670 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/7ba992eba3344a46af01f32012bb6670 2024-11-12T10:54:31,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/284aaea047974f758b7d2a108b43cef7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/284aaea047974f758b7d2a108b43cef7 2024-11-12T10:54:31,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/9889f7efbda84420a37d92f9070ee6d7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/9889f7efbda84420a37d92f9070ee6d7 2024-11-12T10:54:31,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/344ffe34d0304aad89d579e40b846fc6 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/344ffe34d0304aad89d579e40b846fc6 2024-11-12T10:54:31,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/3634c087ec4942d9b5dd4c64717ebd1f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/3634c087ec4942d9b5dd4c64717ebd1f 2024-11-12T10:54:31,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/3badeac3d9c449e6a911519ad7f9ec0d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/3badeac3d9c449e6a911519ad7f9ec0d 2024-11-12T10:54:31,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/6b510213ae3c48ef950dfeef5b3efee9 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/6b510213ae3c48ef950dfeef5b3efee9 2024-11-12T10:54:31,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/fffc16d6e4cc4ebe92c9e0715029c60f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/fffc16d6e4cc4ebe92c9e0715029c60f 2024-11-12T10:54:31,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/9bb31d43613c4d769cb9416e243471b1 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/9bb31d43613c4d769cb9416e243471b1 2024-11-12T10:54:31,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/e1d2771bf20046b0bf1c5b9c738c4726 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/e1d2771bf20046b0bf1c5b9c738c4726 2024-11-12T10:54:31,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/b69d79f31d0942a28af5c3ad836c2315 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/b69d79f31d0942a28af5c3ad836c2315 2024-11-12T10:54:31,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/79e5e1326a6d45909342e6181f4fe98f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/79e5e1326a6d45909342e6181f4fe98f 2024-11-12T10:54:31,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/2746ded2bac549268428738000f01db4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/8cc57cc485d843e9a7ffc58e0337cea7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/81094d7781394eb9bff03bf360e57508, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/9b93b45648d04d74b7b2ed4ed981522a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/5250d73d88594b8a8e77fbdd35d1e0b2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/d5415c0d2927497a823211647e22cb44, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/3af8c4591f224263b9235fbf0c93ff57, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ce7662d41d354a6bb5dfe86f7c5f9b5e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ebc68785ed2940cd972266751f72690a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/2e6a10f10d934528a2270d460e88eb2d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/964097de0e3d44579ea3fabd8a90663b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/1b68e158949d4062acd976589b98e227, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/1c79216cae1245058b67f72ab07f1dcc, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/c85d293220a6484d9f076cfd0343dc6f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/f75d90e7dd53408bbfef4ae8838409a6, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ebb0987570964c99861a319885481b6a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ddd2b216db954e8e92c9ef39094d166f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/8afb18f902914bd4bf66431c30a6fef0] to archive 2024-11-12T10:54:31,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:54:31,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/2746ded2bac549268428738000f01db4 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/2746ded2bac549268428738000f01db4 2024-11-12T10:54:31,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/8cc57cc485d843e9a7ffc58e0337cea7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/8cc57cc485d843e9a7ffc58e0337cea7 2024-11-12T10:54:31,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/81094d7781394eb9bff03bf360e57508 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/81094d7781394eb9bff03bf360e57508 2024-11-12T10:54:31,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/9b93b45648d04d74b7b2ed4ed981522a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/9b93b45648d04d74b7b2ed4ed981522a 2024-11-12T10:54:31,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/5250d73d88594b8a8e77fbdd35d1e0b2 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/5250d73d88594b8a8e77fbdd35d1e0b2 2024-11-12T10:54:31,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/d5415c0d2927497a823211647e22cb44 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/d5415c0d2927497a823211647e22cb44 2024-11-12T10:54:31,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/3af8c4591f224263b9235fbf0c93ff57 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/3af8c4591f224263b9235fbf0c93ff57 2024-11-12T10:54:31,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ce7662d41d354a6bb5dfe86f7c5f9b5e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ce7662d41d354a6bb5dfe86f7c5f9b5e 2024-11-12T10:54:31,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ebc68785ed2940cd972266751f72690a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ebc68785ed2940cd972266751f72690a 2024-11-12T10:54:31,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/2e6a10f10d934528a2270d460e88eb2d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/2e6a10f10d934528a2270d460e88eb2d 2024-11-12T10:54:31,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/964097de0e3d44579ea3fabd8a90663b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/964097de0e3d44579ea3fabd8a90663b 2024-11-12T10:54:31,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/1b68e158949d4062acd976589b98e227 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/1b68e158949d4062acd976589b98e227 2024-11-12T10:54:31,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/1c79216cae1245058b67f72ab07f1dcc to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/1c79216cae1245058b67f72ab07f1dcc 2024-11-12T10:54:31,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/c85d293220a6484d9f076cfd0343dc6f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/c85d293220a6484d9f076cfd0343dc6f 2024-11-12T10:54:31,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/f75d90e7dd53408bbfef4ae8838409a6 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/f75d90e7dd53408bbfef4ae8838409a6 2024-11-12T10:54:31,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ebb0987570964c99861a319885481b6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ebb0987570964c99861a319885481b6a 2024-11-12T10:54:31,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ddd2b216db954e8e92c9ef39094d166f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ddd2b216db954e8e92c9ef39094d166f 2024-11-12T10:54:31,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/8afb18f902914bd4bf66431c30a6fef0 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/8afb18f902914bd4bf66431c30a6fef0 2024-11-12T10:54:31,072 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/recovered.edits/353.seqid, newMaxSeqId=353, maxSeqId=4 2024-11-12T10:54:31,073 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. 2024-11-12T10:54:31,073 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for f3fbab0ede17db49ead8375d789c7d6a: Waiting for close lock at 1731408868770Running coprocessor pre-close hooks at 1731408868770Disabling compacts and flushes for region at 1731408868770Disabling writes for close at 1731408869330 (+560 ms)Obtaining lock to block concurrent updates at 1731408869330Preparing flush snapshotting stores in f3fbab0ede17db49ead8375d789c7d6a at 1731408869330Finished memstore snapshotting TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731408869331 (+1 ms)Flushing stores of TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a. at 1731408869331Flushing f3fbab0ede17db49ead8375d789c7d6a/A: creating writer at 1731408869331Flushing mob file f3fbab0ede17db49ead8375d789c7d6a/A: appending metadata at 1731408869336 (+5 ms)Flushing mob file f3fbab0ede17db49ead8375d789c7d6a/A: closing flushed file at 1731408869336Flushing f3fbab0ede17db49ead8375d789c7d6a/A: appending metadata at 1731408869749 (+413 ms)Flushing f3fbab0ede17db49ead8375d789c7d6a/A: closing flushed file at 1731408869749Flushing f3fbab0ede17db49ead8375d789c7d6a/B: creating writer at 1731408870162 (+413 ms)Flushing f3fbab0ede17db49ead8375d789c7d6a/B: appending metadata at 1731408870166 (+4 ms)Flushing f3fbab0ede17db49ead8375d789c7d6a/B: closing flushed file at 1731408870166Flushing f3fbab0ede17db49ead8375d789c7d6a/C: creating writer at 1731408870581 (+415 ms)Flushing f3fbab0ede17db49ead8375d789c7d6a/C: appending metadata at 1731408870585 (+4 ms)Flushing f3fbab0ede17db49ead8375d789c7d6a/C: closing flushed file at 1731408870585Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79a77f96: reopening flushed file at 1731408870994 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2666b2d9: reopening flushed file at 1731408870999 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21b274e0: reopening flushed file at 1731408871004 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for f3fbab0ede17db49ead8375d789c7d6a in 1681ms, sequenceid=350, compaction requested=true at 1731408871011 (+7 ms)Writing region close event to WAL at 1731408871068 (+57 ms)Running coprocessor post-close hooks at 1731408871073 (+5 ms)Closed at 1731408871073 2024-11-12T10:54:31,075 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:31,076 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=f3fbab0ede17db49ead8375d789c7d6a, regionState=CLOSED 2024-11-12T10:54:31,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:54:31,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-12T10:54:31,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure f3fbab0ede17db49ead8375d789c7d6a, server=aed202db7571,43143,1731408812834 in 2.4610 sec 2024-11-12T10:54:31,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-12T10:54:31,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3fbab0ede17db49ead8375d789c7d6a, UNASSIGN in 2.4680 sec 2024-11-12T10:54:31,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-12T10:54:31,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4730 sec 2024-11-12T10:54:31,088 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408871088"}]},"ts":"1731408871088"} 2024-11-12T10:54:31,090 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T10:54:31,090 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T10:54:31,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4870 sec 2024-11-12T10:54:31,377 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T10:54:32,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T10:54:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T10:54:32,778 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:32,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T10:54:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:32,785 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T10:54:32,788 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:32,791 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,794 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/recovered.edits] 2024-11-12T10:54:32,799 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/1c68cc5a16924039a0f19240a0c3528f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/1c68cc5a16924039a0f19240a0c3528f 2024-11-12T10:54:32,801 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/4450bc82f5d04defafc6fc765145039b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/4450bc82f5d04defafc6fc765145039b 2024-11-12T10:54:32,804 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/5d2fa5f6a69643f694b20b5cb2d39ab7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/5d2fa5f6a69643f694b20b5cb2d39ab7 2024-11-12T10:54:32,806 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/ea5c541c786d44b8b1e074b1de760e59 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/A/ea5c541c786d44b8b1e074b1de760e59 2024-11-12T10:54:32,810 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/0c9dca3387384ba7a9efecba50430c1b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/0c9dca3387384ba7a9efecba50430c1b 2024-11-12T10:54:32,811 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/1d03a0902d0347ad92efecc1d09da7d1 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/1d03a0902d0347ad92efecc1d09da7d1 2024-11-12T10:54:32,812 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/1f6263bf855a408781d92743715174b7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/1f6263bf855a408781d92743715174b7 2024-11-12T10:54:32,813 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/54c42989cf8a4db78f4d1cc80bb73adc to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/B/54c42989cf8a4db78f4d1cc80bb73adc 2024-11-12T10:54:32,815 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/0d58a53f4fa14598bda72a0d41aaa1d6 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/0d58a53f4fa14598bda72a0d41aaa1d6 2024-11-12T10:54:32,816 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/14b7c439ed604adba2058ff236f4ce61 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/14b7c439ed604adba2058ff236f4ce61 2024-11-12T10:54:32,817 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ce6cf7dd6d7e4f8ba6ebf467a5a2e0db to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/ce6cf7dd6d7e4f8ba6ebf467a5a2e0db 2024-11-12T10:54:32,818 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/f8b56ec6b65f490491204faa9b7d2af7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/C/f8b56ec6b65f490491204faa9b7d2af7 2024-11-12T10:54:32,820 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/recovered.edits/353.seqid to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a/recovered.edits/353.seqid 2024-11-12T10:54:32,821 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,821 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T10:54:32,821 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T10:54:32,825 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T10:54:32,829 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122b2ba5406b8d47e1b2f1a42360648706_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122b2ba5406b8d47e1b2f1a42360648706_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,830 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122c0b7759244d46eab367554efaad9b23_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122c0b7759244d46eab367554efaad9b23_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,831 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111233fd7fbfbe5c46198a6d06e07542b22d_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111233fd7fbfbe5c46198a6d06e07542b22d_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,832 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125ea871b61c334fa18d7388439a308171_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125ea871b61c334fa18d7388439a308171_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,833 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126212d86c7bae42429183a913a9c5ce95_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126212d86c7bae42429183a913a9c5ce95_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,834 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111262915d1649b94af0a0895fe718b11725_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111262915d1649b94af0a0895fe718b11725_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,835 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126cc7868c4d1b41f2afc1abddaa242516_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126cc7868c4d1b41f2afc1abddaa242516_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,837 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111271f427c532fa4748b5699e357430f8ff_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111271f427c532fa4748b5699e357430f8ff_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,838 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127f81ae60d0544ba5b02fc98d1e9791d7_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127f81ae60d0544ba5b02fc98d1e9791d7_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,840 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111286099348d0a84f5da15e8aa790b52e4f_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111286099348d0a84f5da15e8aa790b52e4f_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,841 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111290127d1678464a568974a0d967b1a3ab_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111290127d1678464a568974a0d967b1a3ab_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,842 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112925d13bcce804278a08e4f38b908750a_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112925d13bcce804278a08e4f38b908750a_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,843 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a8bebc94745f412aab98aaa4552595c7_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a8bebc94745f412aab98aaa4552595c7_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,844 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c167b4a082274a4f8ebe357bef9ef4ea_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c167b4a082274a4f8ebe357bef9ef4ea_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,845 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d3f7d89270704177bbac60828065e9ed_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d3f7d89270704177bbac60828065e9ed_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,847 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e708afbb5d4746d2a1411b3d7f2bb617_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e708afbb5d4746d2a1411b3d7f2bb617_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,848 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112eeac0b7d219841258abe539c6999ee73_f3fbab0ede17db49ead8375d789c7d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112eeac0b7d219841258abe539c6999ee73_f3fbab0ede17db49ead8375d789c7d6a 2024-11-12T10:54:32,848 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T10:54:32,851 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:32,853 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T10:54:32,855 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T10:54:32,856 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:32,857 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T10:54:32,857 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731408872857"}]},"ts":"9223372036854775807"} 2024-11-12T10:54:32,859 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T10:54:32,859 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f3fbab0ede17db49ead8375d789c7d6a, NAME => 'TestAcidGuarantees,,1731408843020.f3fbab0ede17db49ead8375d789c7d6a.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T10:54:32,859 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T10:54:32,859 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731408872859"}]},"ts":"9223372036854775807"} 2024-11-12T10:54:32,861 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T10:54:32,862 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:32,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 82 msec 2024-11-12T10:54:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T10:54:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T10:54:32,900 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:32,906 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 250) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/cluster_a82bd1d2-3d77-b394-fc32-98907a12be25/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2109716649_22 at /127.0.0.1:38598 [Waiting for operation #433] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1240038058_22 at /127.0.0.1:52656 [Waiting for operation #358] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/cluster_a82bd1d2-3d77-b394-fc32-98907a12be25/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2109716649_22 at /127.0.0.1:55866 [Waiting for operation #305] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=555 (was 680), ProcessCount=11 (was 11), AvailableMemoryMB=2139 (was 2216) 2024-11-12T10:54:32,914 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=555, ProcessCount=11, AvailableMemoryMB=2138 2024-11-12T10:54:32,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T10:54:32,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T10:54:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T10:54:32,919 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T10:54:32,919 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:32,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-12T10:54:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-12T10:54:32,920 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T10:54:32,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742027_1203 (size=1061) 2024-11-12T10:54:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-12T10:54:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-12T10:54:33,334 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98 2024-11-12T10:54:33,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742028_1204 (size=53) 2024-11-12T10:54:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-12T10:54:33,746 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:54:33,746 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 87c17057f5d8c2f7d82f49303921d612, disabling compactions & flushes 2024-11-12T10:54:33,746 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:33,746 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:33,746 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. after waiting 0 ms 2024-11-12T10:54:33,746 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:33,746 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:33,746 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 87c17057f5d8c2f7d82f49303921d612: Waiting for close lock at 1731408873746Disabling compacts and flushes for region at 1731408873746Disabling writes for close at 1731408873746Writing region close event to WAL at 1731408873746Closed at 1731408873746 2024-11-12T10:54:33,748 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T10:54:33,748 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731408873748"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731408873748"}]},"ts":"1731408873748"} 2024-11-12T10:54:33,751 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T10:54:33,752 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T10:54:33,752 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408873752"}]},"ts":"1731408873752"} 2024-11-12T10:54:33,755 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T10:54:33,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87c17057f5d8c2f7d82f49303921d612, ASSIGN}] 2024-11-12T10:54:33,757 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87c17057f5d8c2f7d82f49303921d612, ASSIGN 2024-11-12T10:54:33,758 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87c17057f5d8c2f7d82f49303921d612, ASSIGN; state=OFFLINE, location=aed202db7571,43143,1731408812834; forceNewPlan=false, retain=false 2024-11-12T10:54:33,909 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=87c17057f5d8c2f7d82f49303921d612, regionState=OPENING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:54:33,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87c17057f5d8c2f7d82f49303921d612, ASSIGN because future has completed 2024-11-12T10:54:33,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:54:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-12T10:54:34,068 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:34,068 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:54:34,068 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:34,068 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:54:34,069 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:34,069 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:34,070 INFO [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:34,071 INFO [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:54:34,071 INFO [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87c17057f5d8c2f7d82f49303921d612 columnFamilyName A 2024-11-12T10:54:34,071 DEBUG [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:34,072 INFO [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] regionserver.HStore(327): Store=87c17057f5d8c2f7d82f49303921d612/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:54:34,072 INFO [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:34,073 INFO [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:54:34,073 INFO [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87c17057f5d8c2f7d82f49303921d612 columnFamilyName B 2024-11-12T10:54:34,073 DEBUG [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:34,074 INFO [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] regionserver.HStore(327): Store=87c17057f5d8c2f7d82f49303921d612/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:54:34,074 INFO [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:34,074 INFO [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:54:34,075 INFO [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87c17057f5d8c2f7d82f49303921d612 columnFamilyName C 2024-11-12T10:54:34,075 DEBUG [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:54:34,075 INFO [StoreOpener-87c17057f5d8c2f7d82f49303921d612-1 {}] regionserver.HStore(327): Store=87c17057f5d8c2f7d82f49303921d612/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:54:34,075 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:34,075 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:34,076 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:34,076 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:34,078 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:34,078 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:34,078 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T10:54:34,080 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:34,082 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:54:34,082 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 87c17057f5d8c2f7d82f49303921d612; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66841915, jitterRate=-0.003977850079536438}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T10:54:34,082 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:34,083 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 87c17057f5d8c2f7d82f49303921d612: Running coprocessor pre-open hook at 1731408874069Writing region info on filesystem at 1731408874069Initializing all the Stores at 1731408874070 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408874070Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408874070Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408874070Cleaning up temporary data from old regions at 1731408874078 (+8 ms)Running coprocessor post-open hooks at 1731408874082 (+4 ms)Region opened successfully at 1731408874083 (+1 ms) 2024-11-12T10:54:34,084 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., pid=61, masterSystemTime=1731408874064 2024-11-12T10:54:34,085 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:34,086 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:34,086 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=87c17057f5d8c2f7d82f49303921d612, regionState=OPEN, openSeqNum=2, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:54:34,088 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:54:34,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-12T10:54:34,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 in 178 msec 2024-11-12T10:54:34,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-12T10:54:34,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87c17057f5d8c2f7d82f49303921d612, ASSIGN in 336 msec 2024-11-12T10:54:34,095 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T10:54:34,095 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408874095"}]},"ts":"1731408874095"} 2024-11-12T10:54:34,097 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T10:54:34,098 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T10:54:34,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-11-12T10:54:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-12T10:54:35,077 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:35,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8da64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,078 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:35,078 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:35,079 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:35,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:35,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:35,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8f83a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:35,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:35,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:35,080 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bdfc7e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,081 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:35,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:35,083 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T10:54:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is aed202db7571,45901,1731408812240 2024-11-12T10:54:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@51f34596 2024-11-12T10:54:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T10:54:35,085 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T10:54:35,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,086 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:35,086 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:35,087 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@916f389, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:35,088 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:35,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44064d6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,089 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:35,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:35,089 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e13635, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:35,091 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c562c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,092 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:35,092 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:35,092 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:35,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:35,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:35,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ab57766, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:35,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:35,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:35,093 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:35,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2379e30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:35,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:35,095 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1385ec29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:35,096 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b5f35ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:35,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:35,097 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@402af092, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:35,098 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@383774db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,099 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,099 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:35,099 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:35,100 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:35,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:35,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:35,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@656f69ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:35,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:35,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:35,101 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:35,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59c3c3aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,102 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:35,102 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:35,102 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41a85f24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:35,104 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33711205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,105 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:35,105 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:35,105 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61c83b01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:35,106 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c55c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,107 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:35,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:35,108 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a5218fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:35,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:35,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:35,109 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de2967d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40916ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,110 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:54:35,110 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:54:35,111 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:54:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:54:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:54:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ac103a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:54:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:54:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:35,112 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:54:35,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2efec83b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:54:35,113 DEBUG [Thread-977 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:35,113 DEBUG [Thread-992 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:35,113 DEBUG [Thread-981 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:35,113 DEBUG [Thread-986 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:35,114 DEBUG [Thread-990 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:35,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:35,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:35,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:35,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:35,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:35,114 DEBUG [Thread-983 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:35,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:35,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:35,115 DEBUG [Thread-994 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:35,115 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:35,115 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:35,115 DEBUG [Thread-994 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:35,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:35,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:35,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:35,116 DEBUG [Thread-994 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:35,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:35,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:35,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:35,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:35,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:35,117 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:35,117 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-12T10:54:35,118 DEBUG [Thread-975 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:35,118 DEBUG [Thread-988 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T10:54:35,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:35,119 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:35,119 DEBUG [Thread-988 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:54:35,119 DEBUG [Thread-979 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:54:35,120 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:35,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:35,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:54:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:54:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:54:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:54:35,120 DEBUG [Thread-988 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:54:35,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:35,122 DEBUG [Thread-975 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:54:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:54:35,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:54:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:54:35,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:54:35,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:54:35,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:54:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:35,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T10:54:35,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:35,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:35,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:35,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:35,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:35,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:35,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/ca08d13a588442ec8dee2ca6bced5ebf is 50, key is test_row_0/A:col10/1731408875128/Put/seqid=0 2024-11-12T10:54:35,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408875154, max timestamp=1731408875167Num uniques 859; 2024-11-12T10:54:35,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408875154, max timestamp=1731408875167Num uniques 859; 2024-11-12T10:54:35,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408875154, max timestamp=1731408875167Num uniques 859; 2024-11-12T10:54:35,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:54:35,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:54:35,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:54:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408935165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742029_1205 (size=12001) 2024-11-12T10:54:35,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/ca08d13a588442ec8dee2ca6bced5ebf 2024-11-12T10:54:35,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408935191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408935196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408935197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1731408935198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T10:54:35,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/b5ddafa8537a4f4db61413e9bfbbf343 is 50, key is test_row_0/B:col10/1731408875128/Put/seqid=0 2024-11-12T10:54:35,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742030_1206 (size=12001) 2024-11-12T10:54:35,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T10:54:35,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:35,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:35,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:35,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408935286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408935307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408935307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408935307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1731408935307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T10:54:35,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:35,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:35,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:35,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T10:54:35,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408935497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408935517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408935517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1731408935517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408935517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T10:54:35,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:35,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:35,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:35,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/b5ddafa8537a4f4db61413e9bfbbf343 2024-11-12T10:54:35,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/44a9ef81049943beac7ecc65e33b2c3d is 50, key is test_row_0/C:col10/1731408875128/Put/seqid=0 2024-11-12T10:54:35,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742031_1207 (size=12001) 2024-11-12T10:54:35,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/44a9ef81049943beac7ecc65e33b2c3d 2024-11-12T10:54:35,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/ca08d13a588442ec8dee2ca6bced5ebf as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ca08d13a588442ec8dee2ca6bced5ebf 2024-11-12T10:54:35,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ca08d13a588442ec8dee2ca6bced5ebf, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T10:54:35,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/b5ddafa8537a4f4db61413e9bfbbf343 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/b5ddafa8537a4f4db61413e9bfbbf343 2024-11-12T10:54:35,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/b5ddafa8537a4f4db61413e9bfbbf343, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T10:54:35,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T10:54:35,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:35,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/44a9ef81049943beac7ecc65e33b2c3d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/44a9ef81049943beac7ecc65e33b2c3d 2024-11-12T10:54:35,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:35,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:35,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T10:54:35,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/44a9ef81049943beac7ecc65e33b2c3d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T10:54:35,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 87c17057f5d8c2f7d82f49303921d612 in 624ms, sequenceid=12, compaction requested=false 2024-11-12T10:54:35,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:35,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-12T10:54:35,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:35,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:35,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:35,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:35,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:35,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:35,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/b3e7dd0dadb74e7db92de93677e2cb23 is 50, key is test_row_0/A:col10/1731408875196/Put/seqid=0 2024-11-12T10:54:35,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742032_1208 (size=12001) 2024-11-12T10:54:35,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/b3e7dd0dadb74e7db92de93677e2cb23 2024-11-12T10:54:35,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1731408935827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408935831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408935833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408935835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408935836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/54dc52773a424f20ba2484b130ec32f5 is 50, key is test_row_0/B:col10/1731408875196/Put/seqid=0 2024-11-12T10:54:35,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T10:54:35,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:35,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:35,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:35,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:35,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742033_1209 (size=12001) 2024-11-12T10:54:35,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/54dc52773a424f20ba2484b130ec32f5 2024-11-12T10:54:35,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/cad76160292348b89a961adc70e013f5 is 50, key is test_row_0/C:col10/1731408875196/Put/seqid=0 2024-11-12T10:54:35,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408935947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408935947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408935948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408935948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:35,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742034_1210 (size=12001) 2024-11-12T10:54:36,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T10:54:36,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:36,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408936158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408936158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408936158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408936158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T10:54:36,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:36,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T10:54:36,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T10:54:36,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:36,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1731408936348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/cad76160292348b89a961adc70e013f5 2024-11-12T10:54:36,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/b3e7dd0dadb74e7db92de93677e2cb23 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/b3e7dd0dadb74e7db92de93677e2cb23 2024-11-12T10:54:36,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/b3e7dd0dadb74e7db92de93677e2cb23, entries=150, sequenceid=49, filesize=11.7 K 2024-11-12T10:54:36,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/54dc52773a424f20ba2484b130ec32f5 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/54dc52773a424f20ba2484b130ec32f5 2024-11-12T10:54:36,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/54dc52773a424f20ba2484b130ec32f5, entries=150, sequenceid=49, filesize=11.7 K 2024-11-12T10:54:36,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/cad76160292348b89a961adc70e013f5 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/cad76160292348b89a961adc70e013f5 2024-11-12T10:54:36,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/cad76160292348b89a961adc70e013f5, entries=150, sequenceid=49, filesize=11.7 K 2024-11-12T10:54:36,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 87c17057f5d8c2f7d82f49303921d612 in 586ms, sequenceid=49, compaction requested=false 2024-11-12T10:54:36,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:36,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T10:54:36,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:36,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:36,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:36,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:36,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:36,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:36,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/55b31b4ffb7a467ebf10cd5c0f8c590a is 50, key is test_row_0/A:col10/1731408876469/Put/seqid=0 2024-11-12T10:54:36,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742035_1211 (size=12001) 2024-11-12T10:54:36,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/55b31b4ffb7a467ebf10cd5c0f8c590a 2024-11-12T10:54:36,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T10:54:36,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:36,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/0cc5898576504ce8aad0e11e3d9925da is 50, key is test_row_0/B:col10/1731408876469/Put/seqid=0 2024-11-12T10:54:36,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408936504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408936502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408936521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408936522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742036_1212 (size=12001) 2024-11-12T10:54:36,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/0cc5898576504ce8aad0e11e3d9925da 2024-11-12T10:54:36,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/b2df79da7f37453698f54d0d56d0f648 is 50, key is test_row_0/C:col10/1731408876469/Put/seqid=0 2024-11-12T10:54:36,596 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T10:54:36,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742037_1213 (size=12001) 2024-11-12T10:54:36,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/b2df79da7f37453698f54d0d56d0f648 2024-11-12T10:54:36,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/55b31b4ffb7a467ebf10cd5c0f8c590a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/55b31b4ffb7a467ebf10cd5c0f8c590a 2024-11-12T10:54:36,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/55b31b4ffb7a467ebf10cd5c0f8c590a, entries=150, sequenceid=61, filesize=11.7 K 2024-11-12T10:54:36,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/0cc5898576504ce8aad0e11e3d9925da as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0cc5898576504ce8aad0e11e3d9925da 2024-11-12T10:54:36,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0cc5898576504ce8aad0e11e3d9925da, entries=150, sequenceid=61, filesize=11.7 K 2024-11-12T10:54:36,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/b2df79da7f37453698f54d0d56d0f648 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/b2df79da7f37453698f54d0d56d0f648 2024-11-12T10:54:36,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408936628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408936627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/b2df79da7f37453698f54d0d56d0f648, entries=150, sequenceid=61, filesize=11.7 K 2024-11-12T10:54:36,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 87c17057f5d8c2f7d82f49303921d612 in 169ms, sequenceid=61, compaction requested=true 2024-11-12T10:54:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:36,639 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:36,639 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:36,640 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:36,640 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:36,640 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/A is initiating minor compaction (all files) 2024-11-12T10:54:36,640 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/B is initiating minor compaction (all files) 2024-11-12T10:54:36,640 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/B in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,640 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/A in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,640 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/b5ddafa8537a4f4db61413e9bfbbf343, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/54dc52773a424f20ba2484b130ec32f5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0cc5898576504ce8aad0e11e3d9925da] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=35.2 K 2024-11-12T10:54:36,640 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ca08d13a588442ec8dee2ca6bced5ebf, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/b3e7dd0dadb74e7db92de93677e2cb23, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/55b31b4ffb7a467ebf10cd5c0f8c590a] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=35.2 K 2024-11-12T10:54:36,641 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca08d13a588442ec8dee2ca6bced5ebf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731408875125 2024-11-12T10:54:36,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408876501, max timestamp=1731408876641Num uniques 859; 2024-11-12T10:54:36,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:54:36,644 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting b5ddafa8537a4f4db61413e9bfbbf343, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731408875125 2024-11-12T10:54:36,645 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 54dc52773a424f20ba2484b130ec32f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731408875182 2024-11-12T10:54:36,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=96.16 KB heapSize=282.03 KB 2024-11-12T10:54:36,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:36,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:36,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:36,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:36,650 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cc5898576504ce8aad0e11e3d9925da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731408875833 2024-11-12T10:54:36,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:36,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:36,652 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3e7dd0dadb74e7db92de93677e2cb23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731408875182 2024-11-12T10:54:36,652 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55b31b4ffb7a467ebf10cd5c0f8c590a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731408875833 2024-11-12T10:54:36,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T10:54:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/959bef54260a45a3bf1c67bde29202cf is 50, key is test_row_0/A:col10/1731408876641/Put/seqid=0 2024-11-12T10:54:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,667 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#B#compaction#175 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:36,668 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/ba6d5eaf797b4853a6f73df25999471a is 50, key is test_row_0/B:col10/1731408876469/Put/seqid=0 2024-11-12T10:54:36,675 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#A#compaction#176 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:36,676 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/677d8ab18338468caf6cfc928e5e2d74 is 50, key is test_row_0/A:col10/1731408876469/Put/seqid=0 2024-11-12T10:54:36,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408936693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408936700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742038_1214 (size=16681) 2024-11-12T10:54:36,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742039_1215 (size=12104) 2024-11-12T10:54:36,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742040_1216 (size=12104) 2024-11-12T10:54:36,748 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/677d8ab18338468caf6cfc928e5e2d74 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/677d8ab18338468caf6cfc928e5e2d74 2024-11-12T10:54:36,754 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/A of 87c17057f5d8c2f7d82f49303921d612 into 677d8ab18338468caf6cfc928e5e2d74(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:36,754 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:36,754 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/A, priority=13, startTime=1731408876639; duration=0sec 2024-11-12T10:54:36,754 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:36,754 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:A 2024-11-12T10:54:36,754 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:36,755 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:36,755 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/C is initiating minor compaction (all files) 2024-11-12T10:54:36,756 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/C in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,756 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/44a9ef81049943beac7ecc65e33b2c3d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/cad76160292348b89a961adc70e013f5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/b2df79da7f37453698f54d0d56d0f648] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=35.2 K 2024-11-12T10:54:36,756 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44a9ef81049943beac7ecc65e33b2c3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731408875125 2024-11-12T10:54:36,757 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting cad76160292348b89a961adc70e013f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731408875182 2024-11-12T10:54:36,757 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2df79da7f37453698f54d0d56d0f648, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731408875833 2024-11-12T10:54:36,787 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:36,787 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/34ac4d1748024f3f8a7db5749513928f is 50, key is test_row_0/C:col10/1731408876469/Put/seqid=0 2024-11-12T10:54:36,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408936807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408936807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T10:54:36,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:36,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742041_1217 (size=12104) 2024-11-12T10:54:36,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408936847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408936847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:36,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T10:54:36,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:36,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:36,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:36,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:37,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408937017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408937018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/959bef54260a45a3bf1c67bde29202cf 2024-11-12T10:54:37,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T10:54:37,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:37,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:37,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:37,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:37,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:37,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/50e2406e4bb44e4185a126750ff87bb7 is 50, key is test_row_0/B:col10/1731408876641/Put/seqid=0 2024-11-12T10:54:37,127 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/ba6d5eaf797b4853a6f73df25999471a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/ba6d5eaf797b4853a6f73df25999471a 2024-11-12T10:54:37,133 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/B of 87c17057f5d8c2f7d82f49303921d612 into ba6d5eaf797b4853a6f73df25999471a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:37,134 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:37,134 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/B, priority=13, startTime=1731408876639; duration=0sec 2024-11-12T10:54:37,134 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:37,134 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:B 2024-11-12T10:54:37,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408937157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408937158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742042_1218 (size=12001) 2024-11-12T10:54:37,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/50e2406e4bb44e4185a126750ff87bb7 2024-11-12T10:54:37,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/4f92b410fc174f078a3bd74fe6a33dc9 is 50, key is test_row_0/C:col10/1731408876641/Put/seqid=0 2024-11-12T10:54:37,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742043_1219 (size=12001) 2024-11-12T10:54:37,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/4f92b410fc174f078a3bd74fe6a33dc9 2024-11-12T10:54:37,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/959bef54260a45a3bf1c67bde29202cf as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/959bef54260a45a3bf1c67bde29202cf 2024-11-12T10:54:37,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/959bef54260a45a3bf1c67bde29202cf, entries=250, sequenceid=83, filesize=16.3 K 2024-11-12T10:54:37,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/50e2406e4bb44e4185a126750ff87bb7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/50e2406e4bb44e4185a126750ff87bb7 2024-11-12T10:54:37,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/50e2406e4bb44e4185a126750ff87bb7, entries=150, sequenceid=83, filesize=11.7 K 2024-11-12T10:54:37,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/4f92b410fc174f078a3bd74fe6a33dc9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/4f92b410fc174f078a3bd74fe6a33dc9 2024-11-12T10:54:37,232 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/34ac4d1748024f3f8a7db5749513928f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/34ac4d1748024f3f8a7db5749513928f 2024-11-12T10:54:37,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/4f92b410fc174f078a3bd74fe6a33dc9, entries=150, sequenceid=83, filesize=11.7 K 2024-11-12T10:54:37,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=80.51 KB/82440 for 87c17057f5d8c2f7d82f49303921d612 in 588ms, sequenceid=83, compaction requested=false 2024-11-12T10:54:37,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:37,238 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/C of 87c17057f5d8c2f7d82f49303921d612 into 34ac4d1748024f3f8a7db5749513928f(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:37,238 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:37,238 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/C, priority=13, startTime=1731408876639; duration=0sec 2024-11-12T10:54:37,238 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:37,239 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:C 2024-11-12T10:54:37,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T10:54:37,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:37,275 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T10:54:37,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:37,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:37,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:37,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:37,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:37,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T10:54:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/0d180246831647729b1380abdf4e8974 is 50, key is test_row_0/A:col10/1731408876684/Put/seqid=0 2024-11-12T10:54:37,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742044_1220 (size=12001) 2024-11-12T10:54:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:37,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:37,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1731408937363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408937364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408937364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1731408937477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408937477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408937477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408937667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408937667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408937687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408937687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1731408937687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:37,696 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/0d180246831647729b1380abdf4e8974 2024-11-12T10:54:37,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/2fc814d476e0416d83fbdc245a53de5f is 50, key is test_row_0/B:col10/1731408876684/Put/seqid=0 2024-11-12T10:54:37,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742045_1221 (size=12001) 2024-11-12T10:54:38,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408937997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:38,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:38,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408937998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:38,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:38,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:38,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1731408937998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:38,125 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/2fc814d476e0416d83fbdc245a53de5f 2024-11-12T10:54:38,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/e671c70fa66a4af6b99d438229c8094c is 50, key is test_row_0/C:col10/1731408876684/Put/seqid=0 2024-11-12T10:54:38,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742046_1222 (size=12001) 2024-11-12T10:54:38,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:38,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408938508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:38,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408938509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:38,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1731408938509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:38,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:38,545 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/e671c70fa66a4af6b99d438229c8094c 2024-11-12T10:54:38,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/0d180246831647729b1380abdf4e8974 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/0d180246831647729b1380abdf4e8974 2024-11-12T10:54:38,582 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/0d180246831647729b1380abdf4e8974, entries=150, sequenceid=101, filesize=11.7 K 2024-11-12T10:54:38,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/2fc814d476e0416d83fbdc245a53de5f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/2fc814d476e0416d83fbdc245a53de5f 2024-11-12T10:54:38,590 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/2fc814d476e0416d83fbdc245a53de5f, entries=150, sequenceid=101, filesize=11.7 K 2024-11-12T10:54:38,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/e671c70fa66a4af6b99d438229c8094c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/e671c70fa66a4af6b99d438229c8094c 2024-11-12T10:54:38,602 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/e671c70fa66a4af6b99d438229c8094c, entries=150, sequenceid=101, filesize=11.7 K 2024-11-12T10:54:38,603 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 87c17057f5d8c2f7d82f49303921d612 in 1328ms, sequenceid=101, compaction requested=true 2024-11-12T10:54:38,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:38,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:38,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-12T10:54:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-12T10:54:38,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-12T10:54:38,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4860 sec 2024-11-12T10:54:38,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 3.5000 sec 2024-11-12T10:54:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:38,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T10:54:38,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:38,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:38,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:38,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:38,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:38,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:38,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/756e653768b44c8d96066ce1345e7eaf is 50, key is test_row_0/A:col10/1731408877362/Put/seqid=0 2024-11-12T10:54:38,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408938729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:38,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:38,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408938739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:38,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742047_1223 (size=14341) 2024-11-12T10:54:38,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/756e653768b44c8d96066ce1345e7eaf 2024-11-12T10:54:38,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/5e6fcb71906d4f27a7f75201fb8a2b27 is 50, key is test_row_0/B:col10/1731408877362/Put/seqid=0 2024-11-12T10:54:38,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742048_1224 (size=12001) 2024-11-12T10:54:38,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408938847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:38,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408938857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:39,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408939057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:39,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:39,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408939067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:39,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/5e6fcb71906d4f27a7f75201fb8a2b27 2024-11-12T10:54:39,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/6cb632b99731492f9ea888d81ac39bff is 50, key is test_row_0/C:col10/1731408877362/Put/seqid=0 2024-11-12T10:54:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T10:54:39,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742049_1225 (size=12001) 2024-11-12T10:54:39,288 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:39,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/6cb632b99731492f9ea888d81ac39bff 2024-11-12T10:54:39,289 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-12T10:54:39,291 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T10:54:39,292 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:39,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:39,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/756e653768b44c8d96066ce1345e7eaf as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/756e653768b44c8d96066ce1345e7eaf 2024-11-12T10:54:39,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/756e653768b44c8d96066ce1345e7eaf, entries=200, sequenceid=119, filesize=14.0 K 2024-11-12T10:54:39,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/5e6fcb71906d4f27a7f75201fb8a2b27 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5e6fcb71906d4f27a7f75201fb8a2b27 2024-11-12T10:54:39,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5e6fcb71906d4f27a7f75201fb8a2b27, entries=150, sequenceid=119, filesize=11.7 K 2024-11-12T10:54:39,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/6cb632b99731492f9ea888d81ac39bff as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/6cb632b99731492f9ea888d81ac39bff 2024-11-12T10:54:39,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/6cb632b99731492f9ea888d81ac39bff, entries=150, sequenceid=119, filesize=11.7 K 2024-11-12T10:54:39,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 87c17057f5d8c2f7d82f49303921d612 in 667ms, sequenceid=119, compaction requested=true 2024-11-12T10:54:39,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:39,352 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:39,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:39,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:39,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:39,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:39,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:39,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T10:54:39,352 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:39,354 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55127 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:39,354 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/A is initiating minor compaction (all files) 2024-11-12T10:54:39,354 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/A in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:39,354 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/677d8ab18338468caf6cfc928e5e2d74, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/959bef54260a45a3bf1c67bde29202cf, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/0d180246831647729b1380abdf4e8974, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/756e653768b44c8d96066ce1345e7eaf] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=53.8 K 2024-11-12T10:54:39,355 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:39,355 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/B is initiating minor compaction (all files) 2024-11-12T10:54:39,355 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/B in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:39,355 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/ba6d5eaf797b4853a6f73df25999471a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/50e2406e4bb44e4185a126750ff87bb7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/2fc814d476e0416d83fbdc245a53de5f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5e6fcb71906d4f27a7f75201fb8a2b27] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=47.0 K 2024-11-12T10:54:39,355 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 677d8ab18338468caf6cfc928e5e2d74, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731408875833 2024-11-12T10:54:39,355 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ba6d5eaf797b4853a6f73df25999471a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731408875833 2024-11-12T10:54:39,356 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 959bef54260a45a3bf1c67bde29202cf, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731408876501 2024-11-12T10:54:39,356 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 50e2406e4bb44e4185a126750ff87bb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731408876501 2024-11-12T10:54:39,356 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d180246831647729b1380abdf4e8974, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731408876684 2024-11-12T10:54:39,357 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fc814d476e0416d83fbdc245a53de5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731408876684 2024-11-12T10:54:39,357 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 756e653768b44c8d96066ce1345e7eaf, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731408877361 2024-11-12T10:54:39,357 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e6fcb71906d4f27a7f75201fb8a2b27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731408877361 2024-11-12T10:54:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:39,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T10:54:39,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:39,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:39,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:39,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:39,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:39,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:39,384 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#A#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:39,385 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/f69772e5bd5841f78b9b7a6322368a90 is 50, key is test_row_0/A:col10/1731408877362/Put/seqid=0 2024-11-12T10:54:39,396 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#B#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:39,396 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/0ab08b5b86dd42d990c6b8d8ecf0a4ba is 50, key is test_row_0/B:col10/1731408877362/Put/seqid=0 2024-11-12T10:54:39,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/7d6d6a6f3eee4eeda3d9c98c2ee8cfed is 50, key is test_row_0/A:col10/1731408878723/Put/seqid=0 2024-11-12T10:54:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T10:54:39,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408939436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:39,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408939438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:39,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T10:54:39,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:39,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:39,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:39,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:39,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:39,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742051_1227 (size=12241) 2024-11-12T10:54:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:39,458 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/0ab08b5b86dd42d990c6b8d8ecf0a4ba as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0ab08b5b86dd42d990c6b8d8ecf0a4ba 2024-11-12T10:54:39,467 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/B of 87c17057f5d8c2f7d82f49303921d612 into 0ab08b5b86dd42d990c6b8d8ecf0a4ba(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:39,467 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:39,467 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/B, priority=12, startTime=1731408879352; duration=0sec 2024-11-12T10:54:39,467 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:39,467 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:B 2024-11-12T10:54:39,467 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:39,471 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:39,471 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/C is initiating minor compaction (all files) 2024-11-12T10:54:39,471 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/C in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:39,471 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/34ac4d1748024f3f8a7db5749513928f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/4f92b410fc174f078a3bd74fe6a33dc9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/e671c70fa66a4af6b99d438229c8094c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/6cb632b99731492f9ea888d81ac39bff] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=47.0 K 2024-11-12T10:54:39,471 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 34ac4d1748024f3f8a7db5749513928f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731408875833 2024-11-12T10:54:39,472 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f92b410fc174f078a3bd74fe6a33dc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731408876501 2024-11-12T10:54:39,472 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting e671c70fa66a4af6b99d438229c8094c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731408876684 2024-11-12T10:54:39,473 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cb632b99731492f9ea888d81ac39bff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731408877361 2024-11-12T10:54:39,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742050_1226 (size=12241) 2024-11-12T10:54:39,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742052_1228 (size=14541) 2024-11-12T10:54:39,504 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/f69772e5bd5841f78b9b7a6322368a90 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f69772e5bd5841f78b9b7a6322368a90 2024-11-12T10:54:39,510 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/A of 87c17057f5d8c2f7d82f49303921d612 into f69772e5bd5841f78b9b7a6322368a90(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:39,510 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:39,510 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/A, priority=12, startTime=1731408879352; duration=0sec 2024-11-12T10:54:39,511 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:39,511 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:A 2024-11-12T10:54:39,511 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#C#compaction#189 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:39,512 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/132abf66e1c84122985da7d168f629bc is 50, key is test_row_0/C:col10/1731408877362/Put/seqid=0 2024-11-12T10:54:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1731408939517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408939517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:39,526 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 87c17057f5d8c2f7d82f49303921d612 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:39,527 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 87c17057f5d8c2f7d82f49303921d612 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:39,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408939527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:39,533 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 87c17057f5d8c2f7d82f49303921d612 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:39,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742053_1229 (size=12241) 2024-11-12T10:54:39,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408939547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:39,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408939547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:39,554 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/132abf66e1c84122985da7d168f629bc as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/132abf66e1c84122985da7d168f629bc 2024-11-12T10:54:39,562 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/C of 87c17057f5d8c2f7d82f49303921d612 into 132abf66e1c84122985da7d168f629bc(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:39,562 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:39,562 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/C, priority=12, startTime=1731408879352; duration=0sec 2024-11-12T10:54:39,562 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:39,562 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:C 2024-11-12T10:54:39,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T10:54:39,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:39,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:39,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:39,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:39,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T10:54:39,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T10:54:39,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:39,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:39,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:39,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:39,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:39,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408939757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:39,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408939757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:39,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/7d6d6a6f3eee4eeda3d9c98c2ee8cfed 2024-11-12T10:54:39,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T10:54:39,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:39,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:39,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:39,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:39,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T10:54:39,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/1157a821b1314793b213400ffea65264 is 50, key is test_row_0/B:col10/1731408878723/Put/seqid=0 2024-11-12T10:54:39,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742054_1230 (size=12151) 2024-11-12T10:54:40,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T10:54:40,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:40,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:40,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:40,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:40,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:40,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408940067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:40,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408940067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:40,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T10:54:40,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:40,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:40,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:40,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:40,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:40,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/1157a821b1314793b213400ffea65264 2024-11-12T10:54:40,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T10:54:40,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:40,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:40,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:40,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:40,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:40,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/0ee7173aa2e548d0ae3b203442f15249 is 50, key is test_row_0/C:col10/1731408878723/Put/seqid=0 2024-11-12T10:54:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T10:54:40,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742055_1231 (size=12151) 2024-11-12T10:54:40,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/0ee7173aa2e548d0ae3b203442f15249 2024-11-12T10:54:40,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/7d6d6a6f3eee4eeda3d9c98c2ee8cfed as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/7d6d6a6f3eee4eeda3d9c98c2ee8cfed 2024-11-12T10:54:40,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/7d6d6a6f3eee4eeda3d9c98c2ee8cfed, entries=200, sequenceid=134, filesize=14.2 K 2024-11-12T10:54:40,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/1157a821b1314793b213400ffea65264 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/1157a821b1314793b213400ffea65264 2024-11-12T10:54:40,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/1157a821b1314793b213400ffea65264, entries=150, sequenceid=134, filesize=11.9 K 2024-11-12T10:54:40,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/0ee7173aa2e548d0ae3b203442f15249 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/0ee7173aa2e548d0ae3b203442f15249 2024-11-12T10:54:40,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/0ee7173aa2e548d0ae3b203442f15249, entries=150, sequenceid=134, filesize=11.9 K 2024-11-12T10:54:40,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 87c17057f5d8c2f7d82f49303921d612 in 1105ms, sequenceid=134, compaction requested=false 2024-11-12T10:54:40,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:40,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T10:54:40,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:40,526 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T10:54:40,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:40,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:40,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:40,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:40,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:40,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:40,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/2765d42f7bad4a668cf0fdbeaecb9a70 is 50, key is test_row_0/A:col10/1731408879427/Put/seqid=0 2024-11-12T10:54:40,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742056_1232 (size=12151) 2024-11-12T10:54:40,549 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/2765d42f7bad4a668cf0fdbeaecb9a70 2024-11-12T10:54:40,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/3c98e456dcdc43f98ad384ae698a295c is 50, key is test_row_0/B:col10/1731408879427/Put/seqid=0 2024-11-12T10:54:40,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742057_1233 (size=12151) 2024-11-12T10:54:40,571 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/3c98e456dcdc43f98ad384ae698a295c 2024-11-12T10:54:40,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:40,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:40,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/f7f0e940eec24cc3ab5e6576dd8eed51 is 50, key is test_row_0/C:col10/1731408879427/Put/seqid=0 2024-11-12T10:54:40,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408940618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:40,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408940619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:40,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742058_1234 (size=12151) 2024-11-12T10:54:40,630 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/f7f0e940eec24cc3ab5e6576dd8eed51 2024-11-12T10:54:40,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/2765d42f7bad4a668cf0fdbeaecb9a70 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2765d42f7bad4a668cf0fdbeaecb9a70 2024-11-12T10:54:40,641 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2765d42f7bad4a668cf0fdbeaecb9a70, entries=150, sequenceid=154, filesize=11.9 K 2024-11-12T10:54:40,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/3c98e456dcdc43f98ad384ae698a295c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/3c98e456dcdc43f98ad384ae698a295c 2024-11-12T10:54:40,649 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/3c98e456dcdc43f98ad384ae698a295c, entries=150, sequenceid=154, filesize=11.9 K 2024-11-12T10:54:40,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/f7f0e940eec24cc3ab5e6576dd8eed51 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f7f0e940eec24cc3ab5e6576dd8eed51 2024-11-12T10:54:40,659 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f7f0e940eec24cc3ab5e6576dd8eed51, entries=150, sequenceid=154, filesize=11.9 K 2024-11-12T10:54:40,660 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 87c17057f5d8c2f7d82f49303921d612 in 134ms, sequenceid=154, compaction requested=true 2024-11-12T10:54:40,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:40,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:40,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-12T10:54:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-12T10:54:40,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-12T10:54:40,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3700 sec 2024-11-12T10:54:40,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.3790 sec 2024-11-12T10:54:40,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T10:54:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:40,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:40,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:40,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:40,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:40,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:40,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:40,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/f800d4911c6143eba2b8c258f0f5081d is 50, key is test_row_0/A:col10/1731408880730/Put/seqid=0 2024-11-12T10:54:40,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:40,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408940773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:40,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408940777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:40,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742059_1235 (size=16931) 2024-11-12T10:54:40,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/f800d4911c6143eba2b8c258f0f5081d 2024-11-12T10:54:40,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/9b066c90b6204da18edb2b14ed489468 is 50, key is test_row_0/B:col10/1731408880730/Put/seqid=0 2024-11-12T10:54:40,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742060_1236 (size=12151) 2024-11-12T10:54:40,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408940887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:40,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408940887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:41,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:41,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408941097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:41,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:41,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408941097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:41,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/9b066c90b6204da18edb2b14ed489468 2024-11-12T10:54:41,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/712680c90ed54e4588d1d1a540863e7d is 50, key is test_row_0/C:col10/1731408880730/Put/seqid=0 2024-11-12T10:54:41,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742061_1237 (size=12151) 2024-11-12T10:54:41,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/712680c90ed54e4588d1d1a540863e7d 2024-11-12T10:54:41,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/f800d4911c6143eba2b8c258f0f5081d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f800d4911c6143eba2b8c258f0f5081d 2024-11-12T10:54:41,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f800d4911c6143eba2b8c258f0f5081d, entries=250, sequenceid=171, filesize=16.5 K 2024-11-12T10:54:41,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/9b066c90b6204da18edb2b14ed489468 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/9b066c90b6204da18edb2b14ed489468 2024-11-12T10:54:41,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/9b066c90b6204da18edb2b14ed489468, entries=150, sequenceid=171, filesize=11.9 K 2024-11-12T10:54:41,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/712680c90ed54e4588d1d1a540863e7d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/712680c90ed54e4588d1d1a540863e7d 2024-11-12T10:54:41,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/712680c90ed54e4588d1d1a540863e7d, entries=150, sequenceid=171, filesize=11.9 K 2024-11-12T10:54:41,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 87c17057f5d8c2f7d82f49303921d612 in 607ms, sequenceid=171, compaction requested=true 2024-11-12T10:54:41,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:41,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:41,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:41,339 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:41,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:41,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:41,339 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:41,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:41,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:41,340 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:41,340 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55864 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:41,340 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/B is initiating minor compaction (all files) 2024-11-12T10:54:41,340 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/A is initiating minor compaction (all files) 2024-11-12T10:54:41,341 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/A in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:41,341 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/B in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:41,341 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0ab08b5b86dd42d990c6b8d8ecf0a4ba, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/1157a821b1314793b213400ffea65264, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/3c98e456dcdc43f98ad384ae698a295c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/9b066c90b6204da18edb2b14ed489468] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=47.6 K 2024-11-12T10:54:41,341 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f69772e5bd5841f78b9b7a6322368a90, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/7d6d6a6f3eee4eeda3d9c98c2ee8cfed, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2765d42f7bad4a668cf0fdbeaecb9a70, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f800d4911c6143eba2b8c258f0f5081d] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=54.6 K 2024-11-12T10:54:41,341 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting f69772e5bd5841f78b9b7a6322368a90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731408877361 2024-11-12T10:54:41,341 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ab08b5b86dd42d990c6b8d8ecf0a4ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731408877361 2024-11-12T10:54:41,342 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d6d6a6f3eee4eeda3d9c98c2ee8cfed, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731408878716 2024-11-12T10:54:41,342 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 1157a821b1314793b213400ffea65264, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731408878723 2024-11-12T10:54:41,342 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2765d42f7bad4a668cf0fdbeaecb9a70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731408879415 2024-11-12T10:54:41,342 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c98e456dcdc43f98ad384ae698a295c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731408879415 2024-11-12T10:54:41,343 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b066c90b6204da18edb2b14ed489468, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731408880609 2024-11-12T10:54:41,343 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting f800d4911c6143eba2b8c258f0f5081d, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731408880609 2024-11-12T10:54:41,357 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#B#compaction#198 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:41,357 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/86fbf31313bf4da893d2dbe2f9be7ded is 50, key is test_row_0/B:col10/1731408880730/Put/seqid=0 2024-11-12T10:54:41,358 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#A#compaction#199 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:41,358 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/c9f4602235704925b9fdb1622b52210c is 50, key is test_row_0/A:col10/1731408880730/Put/seqid=0 2024-11-12T10:54:41,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742062_1238 (size=12527) 2024-11-12T10:54:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:41,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T10:54:41,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:41,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:41,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:41,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:41,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:41,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:41,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/704907b93fed462ea78ad6a87586171a is 50, key is test_row_0/A:col10/1731408881420/Put/seqid=0 2024-11-12T10:54:41,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742063_1239 (size=12527) 2024-11-12T10:54:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T10:54:41,437 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:41,437 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-12T10:54:41,440 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T10:54:41,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742064_1240 (size=14541) 2024-11-12T10:54:41,441 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:41,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:41,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408941481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:41,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:41,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408941481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:41,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:41,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:41,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1731408941537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:41,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408941537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:41,544 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 87c17057f5d8c2f7d82f49303921d612 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:41,545 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 87c17057f5d8c2f7d82f49303921d612 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T10:54:41,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408941547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:41,552 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 87c17057f5d8c2f7d82f49303921d612 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:41,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T10:54:41,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:41,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:41,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:41,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:41,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:41,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408941597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408941597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:41,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T10:54:41,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:41,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:41,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:41,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:41,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T10:54:41,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:41,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408941807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408941807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:41,825 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/c9f4602235704925b9fdb1622b52210c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/c9f4602235704925b9fdb1622b52210c 2024-11-12T10:54:41,831 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/A of 87c17057f5d8c2f7d82f49303921d612 into c9f4602235704925b9fdb1622b52210c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:41,831 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:41,831 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/A, priority=12, startTime=1731408881339; duration=0sec 2024-11-12T10:54:41,831 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:41,831 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:A 2024-11-12T10:54:41,831 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:41,836 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:41,836 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/C is initiating minor compaction (all files) 2024-11-12T10:54:41,836 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/C in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:41,840 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/86fbf31313bf4da893d2dbe2f9be7ded as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/86fbf31313bf4da893d2dbe2f9be7ded 2024-11-12T10:54:41,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/704907b93fed462ea78ad6a87586171a 2024-11-12T10:54:41,845 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/132abf66e1c84122985da7d168f629bc, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/0ee7173aa2e548d0ae3b203442f15249, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f7f0e940eec24cc3ab5e6576dd8eed51, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/712680c90ed54e4588d1d1a540863e7d] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=47.6 K 2024-11-12T10:54:41,846 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 132abf66e1c84122985da7d168f629bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731408877361 2024-11-12T10:54:41,848 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ee7173aa2e548d0ae3b203442f15249, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731408878723 2024-11-12T10:54:41,855 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/B of 87c17057f5d8c2f7d82f49303921d612 into 86fbf31313bf4da893d2dbe2f9be7ded(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:41,855 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:41,855 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/B, priority=12, startTime=1731408881339; duration=0sec 2024-11-12T10:54:41,856 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:41,856 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:B 2024-11-12T10:54:41,856 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7f0e940eec24cc3ab5e6576dd8eed51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731408879415 2024-11-12T10:54:41,857 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 712680c90ed54e4588d1d1a540863e7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731408880609 2024-11-12T10:54:41,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/d7708f3d05dd49c7a7295cea3da99667 is 50, key is test_row_0/B:col10/1731408881420/Put/seqid=0 2024-11-12T10:54:41,875 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#C#compaction#202 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:41,876 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/f2e7cd081b2f4ba7b64dc8fe3b29f9ac is 50, key is test_row_0/C:col10/1731408880730/Put/seqid=0 2024-11-12T10:54:41,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T10:54:41,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:41,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:41,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:41,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:41,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:41,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742066_1242 (size=12527) 2024-11-12T10:54:41,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742065_1241 (size=12151) 2024-11-12T10:54:41,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/d7708f3d05dd49c7a7295cea3da99667 2024-11-12T10:54:41,936 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/f2e7cd081b2f4ba7b64dc8fe3b29f9ac as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f2e7cd081b2f4ba7b64dc8fe3b29f9ac 2024-11-12T10:54:41,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/7242c25ac3674028904c73ffb6af2a50 is 50, key is test_row_0/C:col10/1731408881420/Put/seqid=0 2024-11-12T10:54:41,946 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/C of 87c17057f5d8c2f7d82f49303921d612 into f2e7cd081b2f4ba7b64dc8fe3b29f9ac(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:41,946 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:41,946 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/C, priority=12, startTime=1731408881339; duration=0sec 2024-11-12T10:54:41,946 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:41,946 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:C 2024-11-12T10:54:41,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742067_1243 (size=12151) 2024-11-12T10:54:42,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T10:54:42,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:42,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:42,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:42,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:42,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T10:54:42,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408942117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:42,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408942117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:42,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T10:54:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:42,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:42,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T10:54:42,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:42,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:42,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:42,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:42,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:42,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/7242c25ac3674028904c73ffb6af2a50 2024-11-12T10:54:42,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/704907b93fed462ea78ad6a87586171a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/704907b93fed462ea78ad6a87586171a 2024-11-12T10:54:42,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/704907b93fed462ea78ad6a87586171a, entries=200, sequenceid=188, filesize=14.2 K 2024-11-12T10:54:42,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/d7708f3d05dd49c7a7295cea3da99667 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/d7708f3d05dd49c7a7295cea3da99667 2024-11-12T10:54:42,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/d7708f3d05dd49c7a7295cea3da99667, entries=150, sequenceid=188, filesize=11.9 K 2024-11-12T10:54:42,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/7242c25ac3674028904c73ffb6af2a50 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/7242c25ac3674028904c73ffb6af2a50 2024-11-12T10:54:42,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/7242c25ac3674028904c73ffb6af2a50, entries=150, sequenceid=188, filesize=11.9 K 2024-11-12T10:54:42,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 87c17057f5d8c2f7d82f49303921d612 in 1001ms, sequenceid=188, compaction requested=false 2024-11-12T10:54:42,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:42,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T10:54:42,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:42,545 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T10:54:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:42,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/4665530e60444d56aac0419ca5a4eac1 is 50, key is test_row_0/A:col10/1731408881477/Put/seqid=0 2024-11-12T10:54:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T10:54:42,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742068_1244 (size=12151) 2024-11-12T10:54:42,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T10:54:42,591 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-12T10:54:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:42,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:42,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408942669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:42,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408942683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:42,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408942777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:42,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408942787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:42,983 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/4665530e60444d56aac0419ca5a4eac1 2024-11-12T10:54:42,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408942987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:42,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/6be09b33bace4997af0101cbf8d1926a is 50, key is test_row_0/B:col10/1731408881477/Put/seqid=0 2024-11-12T10:54:43,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408942999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:43,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742069_1245 (size=12151) 2024-11-12T10:54:43,030 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/6be09b33bace4997af0101cbf8d1926a 2024-11-12T10:54:43,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/f549a0e5412948db8448f2fd7ef692e3 is 50, key is test_row_0/C:col10/1731408881477/Put/seqid=0 2024-11-12T10:54:43,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742070_1246 (size=12151) 2024-11-12T10:54:43,048 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/f549a0e5412948db8448f2fd7ef692e3 2024-11-12T10:54:43,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/4665530e60444d56aac0419ca5a4eac1 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/4665530e60444d56aac0419ca5a4eac1 2024-11-12T10:54:43,064 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/4665530e60444d56aac0419ca5a4eac1, entries=150, sequenceid=206, filesize=11.9 K 2024-11-12T10:54:43,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/6be09b33bace4997af0101cbf8d1926a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/6be09b33bace4997af0101cbf8d1926a 2024-11-12T10:54:43,077 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/6be09b33bace4997af0101cbf8d1926a, entries=150, sequenceid=206, filesize=11.9 K 2024-11-12T10:54:43,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/f549a0e5412948db8448f2fd7ef692e3 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f549a0e5412948db8448f2fd7ef692e3 2024-11-12T10:54:43,087 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f549a0e5412948db8448f2fd7ef692e3, entries=150, sequenceid=206, filesize=11.9 K 2024-11-12T10:54:43,089 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 87c17057f5d8c2f7d82f49303921d612 in 544ms, sequenceid=206, compaction requested=true 2024-11-12T10:54:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:43,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-12T10:54:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-12T10:54:43,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-12T10:54:43,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6500 sec 2024-11-12T10:54:43,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.6590 sec 2024-11-12T10:54:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:43,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T10:54:43,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:43,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:43,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:43,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:43,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:43,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:43,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/aeae5e734b384eb8a9be7b75a168b728 is 50, key is test_row_0/A:col10/1731408882668/Put/seqid=0 2024-11-12T10:54:43,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742071_1247 (size=14541) 2024-11-12T10:54:43,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/aeae5e734b384eb8a9be7b75a168b728 2024-11-12T10:54:43,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408943332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:43,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:43,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408943333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:43,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:43,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:43,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/219ceab687334ea19ed815302d6335bb is 50, key is test_row_0/B:col10/1731408882668/Put/seqid=0 2024-11-12T10:54:43,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742072_1248 (size=12151) 2024-11-12T10:54:43,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/219ceab687334ea19ed815302d6335bb 2024-11-12T10:54:43,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/20d3cfd541d54af89631f5ab29f9ec33 is 50, key is test_row_0/C:col10/1731408882668/Put/seqid=0 2024-11-12T10:54:43,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742073_1249 (size=12151) 2024-11-12T10:54:43,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/20d3cfd541d54af89631f5ab29f9ec33 2024-11-12T10:54:43,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/aeae5e734b384eb8a9be7b75a168b728 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/aeae5e734b384eb8a9be7b75a168b728 2024-11-12T10:54:43,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/aeae5e734b384eb8a9be7b75a168b728, entries=200, sequenceid=225, filesize=14.2 K 2024-11-12T10:54:43,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/219ceab687334ea19ed815302d6335bb as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/219ceab687334ea19ed815302d6335bb 2024-11-12T10:54:43,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/219ceab687334ea19ed815302d6335bb, entries=150, sequenceid=225, filesize=11.9 K 2024-11-12T10:54:43,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/20d3cfd541d54af89631f5ab29f9ec33 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/20d3cfd541d54af89631f5ab29f9ec33 2024-11-12T10:54:43,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/20d3cfd541d54af89631f5ab29f9ec33, entries=150, sequenceid=225, filesize=11.9 K 2024-11-12T10:54:43,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 87c17057f5d8c2f7d82f49303921d612 in 104ms, sequenceid=225, compaction requested=true 2024-11-12T10:54:43,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:43,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:43,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:43,404 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:43,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:43,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:43,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:43,404 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:43,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:43,405 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:43,405 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53760 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:43,405 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/A is initiating minor compaction (all files) 2024-11-12T10:54:43,405 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/B is initiating minor compaction (all files) 2024-11-12T10:54:43,405 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/B in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:43,405 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/A in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:43,405 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/86fbf31313bf4da893d2dbe2f9be7ded, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/d7708f3d05dd49c7a7295cea3da99667, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/6be09b33bace4997af0101cbf8d1926a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/219ceab687334ea19ed815302d6335bb] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=47.8 K 2024-11-12T10:54:43,405 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/c9f4602235704925b9fdb1622b52210c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/704907b93fed462ea78ad6a87586171a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/4665530e60444d56aac0419ca5a4eac1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/aeae5e734b384eb8a9be7b75a168b728] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=52.5 K 2024-11-12T10:54:43,406 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 86fbf31313bf4da893d2dbe2f9be7ded, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731408880609 2024-11-12T10:54:43,406 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9f4602235704925b9fdb1622b52210c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731408880609 2024-11-12T10:54:43,406 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting d7708f3d05dd49c7a7295cea3da99667, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731408880771 2024-11-12T10:54:43,406 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 704907b93fed462ea78ad6a87586171a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731408880765 2024-11-12T10:54:43,406 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 6be09b33bace4997af0101cbf8d1926a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731408881456 2024-11-12T10:54:43,406 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4665530e60444d56aac0419ca5a4eac1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731408881456 2024-11-12T10:54:43,406 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 219ceab687334ea19ed815302d6335bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731408882648 2024-11-12T10:54:43,406 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeae5e734b384eb8a9be7b75a168b728, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731408882648 2024-11-12T10:54:43,429 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#B#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:43,429 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#A#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:43,430 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/74ddd4d25e7542e38462a514cf7fbe55 is 50, key is test_row_0/A:col10/1731408882668/Put/seqid=0 2024-11-12T10:54:43,430 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/5ecb415f14b542eba414eeedcc98c196 is 50, key is test_row_0/B:col10/1731408882668/Put/seqid=0 2024-11-12T10:54:43,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742074_1250 (size=12663) 2024-11-12T10:54:43,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T10:54:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:43,453 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/5ecb415f14b542eba414eeedcc98c196 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5ecb415f14b542eba414eeedcc98c196 2024-11-12T10:54:43,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:43,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:43,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:43,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:43,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:43,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:43,461 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/B of 87c17057f5d8c2f7d82f49303921d612 into 5ecb415f14b542eba414eeedcc98c196(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:43,461 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:43,461 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/B, priority=12, startTime=1731408883404; duration=0sec 2024-11-12T10:54:43,461 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:43,461 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:B 2024-11-12T10:54:43,461 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:43,463 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:43,463 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/C is initiating minor compaction (all files) 2024-11-12T10:54:43,463 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/C in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:43,463 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f2e7cd081b2f4ba7b64dc8fe3b29f9ac, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/7242c25ac3674028904c73ffb6af2a50, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f549a0e5412948db8448f2fd7ef692e3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/20d3cfd541d54af89631f5ab29f9ec33] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=47.8 K 2024-11-12T10:54:43,463 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting f2e7cd081b2f4ba7b64dc8fe3b29f9ac, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731408880609 2024-11-12T10:54:43,464 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 7242c25ac3674028904c73ffb6af2a50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731408880771 2024-11-12T10:54:43,464 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting f549a0e5412948db8448f2fd7ef692e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731408881456 2024-11-12T10:54:43,464 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 20d3cfd541d54af89631f5ab29f9ec33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731408882648 2024-11-12T10:54:43,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742075_1251 (size=12663) 2024-11-12T10:54:43,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/546ae487207f4ca4964303e4d80169f9 is 50, key is test_row_0/A:col10/1731408883452/Put/seqid=0 2024-11-12T10:54:43,483 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#C#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:43,484 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/78d2f3f891f7416796784b3d6e38cb1d is 50, key is test_row_0/C:col10/1731408882668/Put/seqid=0 2024-11-12T10:54:43,485 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/74ddd4d25e7542e38462a514cf7fbe55 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/74ddd4d25e7542e38462a514cf7fbe55 2024-11-12T10:54:43,494 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/A of 87c17057f5d8c2f7d82f49303921d612 into 74ddd4d25e7542e38462a514cf7fbe55(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:43,494 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:43,495 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/A, priority=12, startTime=1731408883404; duration=0sec 2024-11-12T10:54:43,495 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:43,495 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:A 2024-11-12T10:54:43,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742076_1252 (size=14541) 2024-11-12T10:54:43,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/546ae487207f4ca4964303e4d80169f9 2024-11-12T10:54:43,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/8632d0403431400788c9029185c3b136 is 50, key is test_row_0/B:col10/1731408883452/Put/seqid=0 2024-11-12T10:54:43,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742077_1253 (size=12663) 2024-11-12T10:54:43,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408943510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:43,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408943515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:43,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742078_1254 (size=12151) 2024-11-12T10:54:43,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/8632d0403431400788c9029185c3b136 2024-11-12T10:54:43,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/07ff6e6cad3948399c9ab2ead4013364 is 50, key is test_row_0/C:col10/1731408883452/Put/seqid=0 2024-11-12T10:54:43,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742079_1255 (size=12151) 2024-11-12T10:54:43,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/07ff6e6cad3948399c9ab2ead4013364 2024-11-12T10:54:43,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/546ae487207f4ca4964303e4d80169f9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/546ae487207f4ca4964303e4d80169f9 2024-11-12T10:54:43,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/546ae487207f4ca4964303e4d80169f9, entries=200, sequenceid=240, filesize=14.2 K 2024-11-12T10:54:43,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/8632d0403431400788c9029185c3b136 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8632d0403431400788c9029185c3b136 2024-11-12T10:54:43,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8632d0403431400788c9029185c3b136, entries=150, sequenceid=240, filesize=11.9 K 2024-11-12T10:54:43,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/07ff6e6cad3948399c9ab2ead4013364 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/07ff6e6cad3948399c9ab2ead4013364 2024-11-12T10:54:43,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/07ff6e6cad3948399c9ab2ead4013364, entries=150, sequenceid=240, filesize=11.9 K 2024-11-12T10:54:43,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 87c17057f5d8c2f7d82f49303921d612 in 143ms, sequenceid=240, compaction requested=false 2024-11-12T10:54:43,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T10:54:43,597 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:43,598 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-12T10:54:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T10:54:43,601 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:43,605 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:43,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:43,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T10:54:43,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:43,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:43,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:43,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:43,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:43,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:43,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/54efb5d00a6b4c63a85b526009a60926 is 50, key is test_row_0/A:col10/1731408883504/Put/seqid=0 2024-11-12T10:54:43,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742080_1256 (size=12201) 2024-11-12T10:54:43,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408943659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:43,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408943661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T10:54:43,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T10:54:43,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:43,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:43,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:43,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:43,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408943769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:43,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408943777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:43,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:43,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T10:54:43,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:43,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:43,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:43,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:43,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:43,917 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/78d2f3f891f7416796784b3d6e38cb1d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/78d2f3f891f7416796784b3d6e38cb1d 2024-11-12T10:54:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T10:54:43,923 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/C of 87c17057f5d8c2f7d82f49303921d612 into 78d2f3f891f7416796784b3d6e38cb1d(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:43,923 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:43,923 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/C, priority=12, startTime=1731408883404; duration=0sec 2024-11-12T10:54:43,923 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:43,923 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:C 2024-11-12T10:54:43,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408943977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:43,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408943996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:44,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/54efb5d00a6b4c63a85b526009a60926 2024-11-12T10:54:44,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T10:54:44,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:44,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:44,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:44,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:44,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:44,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/075ce11dbefd472f97651dc05e73d4ec is 50, key is test_row_0/B:col10/1731408883504/Put/seqid=0 2024-11-12T10:54:44,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742081_1257 (size=12201) 2024-11-12T10:54:44,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T10:54:44,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:44,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:44,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:44,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:44,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T10:54:44,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408944288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:44,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:44,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408944307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:44,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T10:54:44,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:44,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:44,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:44,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:44,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:44,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/075ce11dbefd472f97651dc05e73d4ec 2024-11-12T10:54:44,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/ef98c0f363a64a08968ca835ee97ea77 is 50, key is test_row_0/C:col10/1731408883504/Put/seqid=0 2024-11-12T10:54:44,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T10:54:44,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:44,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:44,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:44,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:44,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:44,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742082_1258 (size=12201) 2024-11-12T10:54:44,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/ef98c0f363a64a08968ca835ee97ea77 2024-11-12T10:54:44,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/54efb5d00a6b4c63a85b526009a60926 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/54efb5d00a6b4c63a85b526009a60926 2024-11-12T10:54:44,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/54efb5d00a6b4c63a85b526009a60926, entries=150, sequenceid=260, filesize=11.9 K 2024-11-12T10:54:44,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/075ce11dbefd472f97651dc05e73d4ec as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/075ce11dbefd472f97651dc05e73d4ec 2024-11-12T10:54:44,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/075ce11dbefd472f97651dc05e73d4ec, entries=150, sequenceid=260, filesize=11.9 K 2024-11-12T10:54:44,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/ef98c0f363a64a08968ca835ee97ea77 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/ef98c0f363a64a08968ca835ee97ea77 2024-11-12T10:54:44,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/ef98c0f363a64a08968ca835ee97ea77, entries=150, sequenceid=260, filesize=11.9 K 2024-11-12T10:54:44,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 87c17057f5d8c2f7d82f49303921d612 in 962ms, sequenceid=260, compaction requested=true 2024-11-12T10:54:44,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:44,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:44,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:44,584 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:44,584 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:44,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:44,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:44,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:44,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:44,585 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:44,585 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/B is initiating minor compaction (all files) 2024-11-12T10:54:44,585 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/B in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:44,585 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5ecb415f14b542eba414eeedcc98c196, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8632d0403431400788c9029185c3b136, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/075ce11dbefd472f97651dc05e73d4ec] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=36.1 K 2024-11-12T10:54:44,585 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:44,585 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/A is initiating minor compaction (all files) 2024-11-12T10:54:44,585 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/A in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:44,585 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/74ddd4d25e7542e38462a514cf7fbe55, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/546ae487207f4ca4964303e4d80169f9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/54efb5d00a6b4c63a85b526009a60926] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=38.5 K 2024-11-12T10:54:44,586 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74ddd4d25e7542e38462a514cf7fbe55, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731408882648 2024-11-12T10:54:44,586 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ecb415f14b542eba414eeedcc98c196, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731408882648 2024-11-12T10:54:44,586 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 546ae487207f4ca4964303e4d80169f9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731408883316 2024-11-12T10:54:44,586 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 8632d0403431400788c9029185c3b136, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731408883316 2024-11-12T10:54:44,587 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 075ce11dbefd472f97651dc05e73d4ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731408883487 2024-11-12T10:54:44,587 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54efb5d00a6b4c63a85b526009a60926, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731408883487 2024-11-12T10:54:44,604 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:44,605 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/fbb43a005fc54f0f8a53987800fdf7f0 is 50, key is test_row_0/B:col10/1731408883504/Put/seqid=0 2024-11-12T10:54:44,608 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#A#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:44,609 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/578a032f068c4579868a031c24309cd7 is 50, key is test_row_0/A:col10/1731408883504/Put/seqid=0 2024-11-12T10:54:44,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742083_1259 (size=12815) 2024-11-12T10:54:44,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T10:54:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:44,680 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T10:54:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:44,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742084_1260 (size=12815) 2024-11-12T10:54:44,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/2ca17df3f7494c01822710b895f51410 is 50, key is test_row_0/A:col10/1731408883659/Put/seqid=0 2024-11-12T10:54:44,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742085_1261 (size=12301) 2024-11-12T10:54:44,719 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/2ca17df3f7494c01822710b895f51410 2024-11-12T10:54:44,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/5001fa94586942d5865469449e9a2c97 is 50, key is test_row_0/B:col10/1731408883659/Put/seqid=0 2024-11-12T10:54:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T10:54:44,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742086_1262 (size=12301) 2024-11-12T10:54:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:44,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:44,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408944860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:44,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408944862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:44,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:44,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408944977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:44,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:44,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408944977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:45,080 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/fbb43a005fc54f0f8a53987800fdf7f0 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/fbb43a005fc54f0f8a53987800fdf7f0 2024-11-12T10:54:45,085 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/B of 87c17057f5d8c2f7d82f49303921d612 into fbb43a005fc54f0f8a53987800fdf7f0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:45,085 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:45,085 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/B, priority=13, startTime=1731408884584; duration=0sec 2024-11-12T10:54:45,085 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:45,085 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:B 2024-11-12T10:54:45,085 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:45,086 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:45,086 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/C is initiating minor compaction (all files) 2024-11-12T10:54:45,086 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/C in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:45,087 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/78d2f3f891f7416796784b3d6e38cb1d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/07ff6e6cad3948399c9ab2ead4013364, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/ef98c0f363a64a08968ca835ee97ea77] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=36.1 K 2024-11-12T10:54:45,087 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 78d2f3f891f7416796784b3d6e38cb1d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731408882648 2024-11-12T10:54:45,087 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 07ff6e6cad3948399c9ab2ead4013364, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731408883316 2024-11-12T10:54:45,088 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ef98c0f363a64a08968ca835ee97ea77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731408883487 2024-11-12T10:54:45,101 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/578a032f068c4579868a031c24309cd7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/578a032f068c4579868a031c24309cd7 2024-11-12T10:54:45,109 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#C#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:45,110 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/A of 87c17057f5d8c2f7d82f49303921d612 into 578a032f068c4579868a031c24309cd7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:45,110 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:45,110 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/498ca2207a2a43b4889b1d252d39d3d2 is 50, key is test_row_0/C:col10/1731408883504/Put/seqid=0 2024-11-12T10:54:45,110 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/A, priority=13, startTime=1731408884583; duration=0sec 2024-11-12T10:54:45,110 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:45,110 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:A 2024-11-12T10:54:45,143 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/5001fa94586942d5865469449e9a2c97 2024-11-12T10:54:45,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/d60adf2159a341c6ad80fcde8a14059a is 50, key is test_row_0/C:col10/1731408883659/Put/seqid=0 2024-11-12T10:54:45,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742087_1263 (size=12815) 2024-11-12T10:54:45,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742088_1264 (size=12301) 2024-11-12T10:54:45,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:45,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408945188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408945188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:45,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408945497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:45,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408945497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:45,562 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/498ca2207a2a43b4889b1d252d39d3d2 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/498ca2207a2a43b4889b1d252d39d3d2 2024-11-12T10:54:45,565 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/d60adf2159a341c6ad80fcde8a14059a 2024-11-12T10:54:45,567 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/C of 87c17057f5d8c2f7d82f49303921d612 into 498ca2207a2a43b4889b1d252d39d3d2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:45,567 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:45,567 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/C, priority=13, startTime=1731408884584; duration=0sec 2024-11-12T10:54:45,567 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:45,567 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:C 2024-11-12T10:54:45,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731408945567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:45,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1731408945567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:45,570 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 87c17057f5d8c2f7d82f49303921d612 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:45,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/2ca17df3f7494c01822710b895f51410 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2ca17df3f7494c01822710b895f51410 2024-11-12T10:54:45,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:45,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:45,571 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 87c17057f5d8c2f7d82f49303921d612 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:45,577 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2ca17df3f7494c01822710b895f51410, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T10:54:45,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/5001fa94586942d5865469449e9a2c97 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5001fa94586942d5865469449e9a2c97 2024-11-12T10:54:45,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408945577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:45,581 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 87c17057f5d8c2f7d82f49303921d612 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:45,583 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5001fa94586942d5865469449e9a2c97, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T10:54:45,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/d60adf2159a341c6ad80fcde8a14059a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d60adf2159a341c6ad80fcde8a14059a 2024-11-12T10:54:45,590 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d60adf2159a341c6ad80fcde8a14059a, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T10:54:45,591 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 87c17057f5d8c2f7d82f49303921d612 in 911ms, sequenceid=275, compaction requested=false 2024-11-12T10:54:45,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:45,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:45,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-12T10:54:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-12T10:54:45,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-12T10:54:45,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9880 sec 2024-11-12T10:54:45,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.9990 sec 2024-11-12T10:54:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T10:54:45,747 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:45,747 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-12T10:54:45,749 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:45,750 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:45,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T10:54:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T10:54:45,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T10:54:45,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:45,903 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T10:54:45,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:45,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:45,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/10b300c8f62044be868ca580459d5079 is 50, key is test_row_0/A:col10/1731408884859/Put/seqid=0 2024-11-12T10:54:45,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742089_1265 (size=12301) 2024-11-12T10:54:45,914 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/10b300c8f62044be868ca580459d5079 2024-11-12T10:54:45,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/b02d7d4de3214eb2880a45ddb448bf77 is 50, key is test_row_0/B:col10/1731408884859/Put/seqid=0 2024-11-12T10:54:45,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742090_1266 (size=12301) 2024-11-12T10:54:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:46,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:46,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408946027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:46,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408946031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:46,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T10:54:46,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408946137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:46,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:46,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408946147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:46,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:46,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408946347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:46,350 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/b02d7d4de3214eb2880a45ddb448bf77 2024-11-12T10:54:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:46,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/0a5c6197950441ed8761a0b3f6dccdf7 is 50, key is test_row_0/C:col10/1731408884859/Put/seqid=0 2024-11-12T10:54:46,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408946358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:46,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:46,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742091_1267 (size=12301) 2024-11-12T10:54:46,368 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/0a5c6197950441ed8761a0b3f6dccdf7 2024-11-12T10:54:46,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/10b300c8f62044be868ca580459d5079 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/10b300c8f62044be868ca580459d5079 2024-11-12T10:54:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T10:54:46,379 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/10b300c8f62044be868ca580459d5079, entries=150, sequenceid=297, filesize=12.0 K 2024-11-12T10:54:46,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/b02d7d4de3214eb2880a45ddb448bf77 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/b02d7d4de3214eb2880a45ddb448bf77 2024-11-12T10:54:46,385 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/b02d7d4de3214eb2880a45ddb448bf77, entries=150, sequenceid=297, filesize=12.0 K 2024-11-12T10:54:46,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/0a5c6197950441ed8761a0b3f6dccdf7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/0a5c6197950441ed8761a0b3f6dccdf7 2024-11-12T10:54:46,392 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/0a5c6197950441ed8761a0b3f6dccdf7, entries=150, sequenceid=297, filesize=12.0 K 2024-11-12T10:54:46,394 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 87c17057f5d8c2f7d82f49303921d612 in 491ms, sequenceid=297, compaction requested=true 2024-11-12T10:54:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-12T10:54:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-12T10:54:46,397 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-12T10:54:46,397 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 646 msec 2024-11-12T10:54:46,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 651 msec 2024-11-12T10:54:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:46,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T10:54:46,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:46,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:46,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:46,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:46,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:46,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:46,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/2b9f78d2b5214046acd8e85686880e65 is 50, key is test_row_0/A:col10/1731408886026/Put/seqid=0 2024-11-12T10:54:46,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742092_1268 (size=14741) 2024-11-12T10:54:46,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408946742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:46,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408946742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:46,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:46,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408946857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:46,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408946857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T10:54:46,887 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:46,888 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:46,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-12T10:54:46,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T10:54:46,892 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:46,893 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:46,893 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T10:54:47,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T10:54:47,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:47,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:47,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:47,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408947067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:47,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408947067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:47,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:47,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:47,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:47,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/2b9f78d2b5214046acd8e85686880e65 2024-11-12T10:54:47,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/753de3713b1e484d85c6fe357defa881 is 50, key is test_row_0/B:col10/1731408886026/Put/seqid=0 2024-11-12T10:54:47,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742093_1269 (size=12301) 2024-11-12T10:54:47,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T10:54:47,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:47,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:47,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:47,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T10:54:47,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T10:54:47,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:47,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:47,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:47,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408947377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:47,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408947377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:47,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T10:54:47,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:47,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:47,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:47,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/753de3713b1e484d85c6fe357defa881 2024-11-12T10:54:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T10:54:47,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/31fc39be1adb449ca200b746235491ac is 50, key is test_row_0/C:col10/1731408886026/Put/seqid=0 2024-11-12T10:54:47,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742094_1270 (size=12301) 2024-11-12T10:54:47,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T10:54:47,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:47,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:47,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:47,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T10:54:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:47,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408947887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:47,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408947896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:47,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:47,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T10:54:47,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:47,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:47,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:47,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:47,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/31fc39be1adb449ca200b746235491ac 2024-11-12T10:54:47,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/2b9f78d2b5214046acd8e85686880e65 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2b9f78d2b5214046acd8e85686880e65 2024-11-12T10:54:47,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2b9f78d2b5214046acd8e85686880e65, entries=200, sequenceid=311, filesize=14.4 K 2024-11-12T10:54:48,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/753de3713b1e484d85c6fe357defa881 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/753de3713b1e484d85c6fe357defa881 2024-11-12T10:54:48,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/753de3713b1e484d85c6fe357defa881, entries=150, sequenceid=311, filesize=12.0 K 2024-11-12T10:54:48,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/31fc39be1adb449ca200b746235491ac as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/31fc39be1adb449ca200b746235491ac 2024-11-12T10:54:48,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/31fc39be1adb449ca200b746235491ac, entries=150, sequenceid=311, filesize=12.0 K 2024-11-12T10:54:48,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 87c17057f5d8c2f7d82f49303921d612 in 1351ms, sequenceid=311, compaction requested=true 2024-11-12T10:54:48,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:48,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:48,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:48,012 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:48,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:48,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:48,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:48,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T10:54:48,012 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:48,013 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:48,013 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/A is initiating minor compaction (all files) 2024-11-12T10:54:48,013 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49718 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:48,013 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/A in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:48,013 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/B is initiating minor compaction (all files) 2024-11-12T10:54:48,013 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/B in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:48,013 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/578a032f068c4579868a031c24309cd7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2ca17df3f7494c01822710b895f51410, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/10b300c8f62044be868ca580459d5079, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2b9f78d2b5214046acd8e85686880e65] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=50.9 K 2024-11-12T10:54:48,013 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/fbb43a005fc54f0f8a53987800fdf7f0, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5001fa94586942d5865469449e9a2c97, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/b02d7d4de3214eb2880a45ddb448bf77, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/753de3713b1e484d85c6fe357defa881] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=48.6 K 2024-11-12T10:54:48,013 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 578a032f068c4579868a031c24309cd7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731408883487 2024-11-12T10:54:48,013 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbb43a005fc54f0f8a53987800fdf7f0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731408883487 2024-11-12T10:54:48,014 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ca17df3f7494c01822710b895f51410, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731408883643 2024-11-12T10:54:48,014 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5001fa94586942d5865469449e9a2c97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731408883643 2024-11-12T10:54:48,014 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 10b300c8f62044be868ca580459d5079, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731408884809 2024-11-12T10:54:48,014 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting b02d7d4de3214eb2880a45ddb448bf77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731408884809 2024-11-12T10:54:48,014 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b9f78d2b5214046acd8e85686880e65, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731408886025 2024-11-12T10:54:48,014 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 753de3713b1e484d85c6fe357defa881, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731408886025 2024-11-12T10:54:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T10:54:48,037 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:48,037 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#B#compaction#232 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:48,037 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/ae2266ef3c0540d589a5cc429ee8fc84 is 50, key is test_row_0/A:col10/1731408886026/Put/seqid=0 2024-11-12T10:54:48,037 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/a2c135e154d34289962915e8fbf873ad is 50, key is test_row_0/B:col10/1731408886026/Put/seqid=0 2024-11-12T10:54:48,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742095_1271 (size=13051) 2024-11-12T10:54:48,078 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/ae2266ef3c0540d589a5cc429ee8fc84 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ae2266ef3c0540d589a5cc429ee8fc84 2024-11-12T10:54:48,084 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/A of 87c17057f5d8c2f7d82f49303921d612 into ae2266ef3c0540d589a5cc429ee8fc84(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:48,085 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:48,085 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/A, priority=12, startTime=1731408888012; duration=0sec 2024-11-12T10:54:48,085 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:48,085 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:A 2024-11-12T10:54:48,085 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:48,087 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49718 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:48,087 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/C is initiating minor compaction (all files) 2024-11-12T10:54:48,087 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/C in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:48,087 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/498ca2207a2a43b4889b1d252d39d3d2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d60adf2159a341c6ad80fcde8a14059a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/0a5c6197950441ed8761a0b3f6dccdf7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/31fc39be1adb449ca200b746235491ac] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=48.6 K 2024-11-12T10:54:48,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742096_1272 (size=13051) 2024-11-12T10:54:48,088 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 498ca2207a2a43b4889b1d252d39d3d2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731408883487 2024-11-12T10:54:48,088 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting d60adf2159a341c6ad80fcde8a14059a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731408883643 2024-11-12T10:54:48,088 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a5c6197950441ed8761a0b3f6dccdf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731408884809 2024-11-12T10:54:48,089 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 31fc39be1adb449ca200b746235491ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731408886025 2024-11-12T10:54:48,103 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#C#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:48,104 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/eade2b6d477a49ea9e7005dcbc258bad is 50, key is test_row_0/C:col10/1731408886026/Put/seqid=0 2024-11-12T10:54:48,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742097_1273 (size=13051) 2024-11-12T10:54:48,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T10:54:48,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:48,120 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T10:54:48,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:48,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:48,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:48,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:48,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:48,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:48,122 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/eade2b6d477a49ea9e7005dcbc258bad as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/eade2b6d477a49ea9e7005dcbc258bad 2024-11-12T10:54:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/0bdec34676714edd8431399782de72b8 is 50, key is test_row_0/A:col10/1731408886713/Put/seqid=0 2024-11-12T10:54:48,134 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/C of 87c17057f5d8c2f7d82f49303921d612 into eade2b6d477a49ea9e7005dcbc258bad(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:48,134 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:48,134 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/C, priority=12, startTime=1731408888012; duration=0sec 2024-11-12T10:54:48,134 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:48,134 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:C 2024-11-12T10:54:48,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742098_1274 (size=12301) 2024-11-12T10:54:48,495 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/a2c135e154d34289962915e8fbf873ad as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/a2c135e154d34289962915e8fbf873ad 2024-11-12T10:54:48,502 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/B of 87c17057f5d8c2f7d82f49303921d612 into a2c135e154d34289962915e8fbf873ad(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:48,502 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:48,502 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/B, priority=12, startTime=1731408888012; duration=0sec 2024-11-12T10:54:48,502 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:48,502 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:B 2024-11-12T10:54:48,539 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/0bdec34676714edd8431399782de72b8 2024-11-12T10:54:48,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/21f3ada9b46e447ebdcbcd3748456d28 is 50, key is test_row_0/B:col10/1731408886713/Put/seqid=0 2024-11-12T10:54:48,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742099_1275 (size=12301) 2024-11-12T10:54:48,573 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/21f3ada9b46e447ebdcbcd3748456d28 2024-11-12T10:54:48,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/80c68b2c265a4243b6da816d140a2c00 is 50, key is test_row_0/C:col10/1731408886713/Put/seqid=0 2024-11-12T10:54:48,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742100_1276 (size=12301) 2024-11-12T10:54:48,593 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/80c68b2c265a4243b6da816d140a2c00 2024-11-12T10:54:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/0bdec34676714edd8431399782de72b8 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/0bdec34676714edd8431399782de72b8 2024-11-12T10:54:48,604 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/0bdec34676714edd8431399782de72b8, entries=150, sequenceid=331, filesize=12.0 K 2024-11-12T10:54:48,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/21f3ada9b46e447ebdcbcd3748456d28 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/21f3ada9b46e447ebdcbcd3748456d28 2024-11-12T10:54:48,608 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/21f3ada9b46e447ebdcbcd3748456d28, entries=150, sequenceid=331, filesize=12.0 K 2024-11-12T10:54:48,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/80c68b2c265a4243b6da816d140a2c00 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/80c68b2c265a4243b6da816d140a2c00 2024-11-12T10:54:48,614 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/80c68b2c265a4243b6da816d140a2c00, entries=150, sequenceid=331, filesize=12.0 K 2024-11-12T10:54:48,616 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 87c17057f5d8c2f7d82f49303921d612 in 496ms, sequenceid=331, compaction requested=false 2024-11-12T10:54:48,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:48,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:48,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-12T10:54:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-12T10:54:48,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-12T10:54:48,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7250 sec 2024-11-12T10:54:48,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.7320 sec 2024-11-12T10:54:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:48,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T10:54:48,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:48,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/1b226b206f60442292b56047bd09feff is 50, key is test_row_0/A:col10/1731408888919/Put/seqid=0 2024-11-12T10:54:48,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742101_1277 (size=12301) 2024-11-12T10:54:48,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/1b226b206f60442292b56047bd09feff 2024-11-12T10:54:48,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/84db52a7300f4101beb958c49f33bf68 is 50, key is test_row_0/B:col10/1731408888919/Put/seqid=0 2024-11-12T10:54:48,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742102_1278 (size=12301) 2024-11-12T10:54:48,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731408888949, max timestamp=1731408888979Num uniques 859; 2024-11-12T10:54:48,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731408888949, max timestamp=1731408888979Num uniques 859; 2024-11-12T10:54:48,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731408888949, max timestamp=1731408888979Num uniques 859; 2024-11-12T10:54:48,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T10:54:48,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T10:54:48,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T10:54:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T10:54:49,037 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:49,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408949031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:49,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,038 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408949031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-12T10:54:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T10:54:49,039 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:49,040 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:49,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T10:54:49,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408949147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408949147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:49,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T10:54:49,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:49,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:49,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:49,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T10:54:49,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:49,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:49,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:49,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/84db52a7300f4101beb958c49f33bf68 2024-11-12T10:54:49,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T10:54:49,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/7f512fbc4e7140dc8567cd697bf942c7 is 50, key is test_row_0/C:col10/1731408888919/Put/seqid=0 2024-11-12T10:54:49,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408949357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:49,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408949362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:49,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:49,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742103_1279 (size=12301) 2024-11-12T10:54:49,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T10:54:49,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:49,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:49,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:49,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T10:54:49,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:49,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:49,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:49,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T10:54:49,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408949667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:49,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408949677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:49,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:49,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:49,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/7f512fbc4e7140dc8567cd697bf942c7 2024-11-12T10:54:49,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/1b226b206f60442292b56047bd09feff as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1b226b206f60442292b56047bd09feff 2024-11-12T10:54:49,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1b226b206f60442292b56047bd09feff, entries=150, sequenceid=344, filesize=12.0 K 2024-11-12T10:54:49,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/84db52a7300f4101beb958c49f33bf68 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/84db52a7300f4101beb958c49f33bf68 2024-11-12T10:54:49,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T10:54:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:49,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:49,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/84db52a7300f4101beb958c49f33bf68, entries=150, sequenceid=344, filesize=12.0 K 2024-11-12T10:54:49,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/7f512fbc4e7140dc8567cd697bf942c7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/7f512fbc4e7140dc8567cd697bf942c7 2024-11-12T10:54:49,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/7f512fbc4e7140dc8567cd697bf942c7, entries=150, sequenceid=344, filesize=12.0 K 2024-11-12T10:54:49,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 87c17057f5d8c2f7d82f49303921d612 in 902ms, sequenceid=344, compaction requested=true 2024-11-12T10:54:49,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:49,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:49,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:49,822 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:49,822 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:49,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:49,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:49,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:49,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:49,824 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:49,824 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:49,824 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/B is initiating minor compaction (all files) 2024-11-12T10:54:49,824 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/A is initiating minor compaction (all files) 2024-11-12T10:54:49,824 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/A in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:49,824 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/B in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:49,824 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/a2c135e154d34289962915e8fbf873ad, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/21f3ada9b46e447ebdcbcd3748456d28, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/84db52a7300f4101beb958c49f33bf68] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=36.8 K 2024-11-12T10:54:49,824 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ae2266ef3c0540d589a5cc429ee8fc84, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/0bdec34676714edd8431399782de72b8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1b226b206f60442292b56047bd09feff] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=36.8 K 2024-11-12T10:54:49,825 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting a2c135e154d34289962915e8fbf873ad, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731408886025 2024-11-12T10:54:49,825 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae2266ef3c0540d589a5cc429ee8fc84, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731408886025 2024-11-12T10:54:49,825 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 21f3ada9b46e447ebdcbcd3748456d28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731408886705 2024-11-12T10:54:49,825 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bdec34676714edd8431399782de72b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731408886705 2024-11-12T10:54:49,825 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 84db52a7300f4101beb958c49f33bf68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731408888905 2024-11-12T10:54:49,826 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b226b206f60442292b56047bd09feff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731408888905 2024-11-12T10:54:49,837 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:49,837 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/8740ab04240844d395166a02a40197da is 50, key is test_row_0/B:col10/1731408888919/Put/seqid=0 2024-11-12T10:54:49,845 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:49,846 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/ff3e0f5dd96643829adbc2b3cf17d889 is 50, key is test_row_0/A:col10/1731408888919/Put/seqid=0 2024-11-12T10:54:49,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742104_1280 (size=13153) 2024-11-12T10:54:49,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742105_1281 (size=13153) 2024-11-12T10:54:49,901 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/ff3e0f5dd96643829adbc2b3cf17d889 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ff3e0f5dd96643829adbc2b3cf17d889 2024-11-12T10:54:49,908 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/A of 87c17057f5d8c2f7d82f49303921d612 into ff3e0f5dd96643829adbc2b3cf17d889(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:49,908 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:49,908 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/A, priority=13, startTime=1731408889822; duration=0sec 2024-11-12T10:54:49,908 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:49,908 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:A 2024-11-12T10:54:49,908 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:49,909 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:49,910 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/C is initiating minor compaction (all files) 2024-11-12T10:54:49,910 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/C in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:49,910 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/eade2b6d477a49ea9e7005dcbc258bad, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/80c68b2c265a4243b6da816d140a2c00, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/7f512fbc4e7140dc8567cd697bf942c7] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=36.8 K 2024-11-12T10:54:49,912 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting eade2b6d477a49ea9e7005dcbc258bad, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731408886025 2024-11-12T10:54:49,912 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80c68b2c265a4243b6da816d140a2c00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731408886705 2024-11-12T10:54:49,913 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f512fbc4e7140dc8567cd697bf942c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731408888905 2024-11-12T10:54:49,923 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:49,925 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/d28a8631929c4c2bad7d1b9ce73218e4 is 50, key is test_row_0/C:col10/1731408888919/Put/seqid=0 2024-11-12T10:54:49,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742106_1282 (size=13153) 2024-11-12T10:54:49,948 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/d28a8631929c4c2bad7d1b9ce73218e4 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d28a8631929c4c2bad7d1b9ce73218e4 2024-11-12T10:54:49,955 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/C of 87c17057f5d8c2f7d82f49303921d612 into d28a8631929c4c2bad7d1b9ce73218e4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:49,955 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:49,955 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/C, priority=13, startTime=1731408889822; duration=0sec 2024-11-12T10:54:49,955 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:49,955 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:C 2024-11-12T10:54:49,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T10:54:49,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:49,965 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-12T10:54:49,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:49,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:49,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:49,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:49,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:49,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:54:49,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/1743439c7c944498ab9459c1d10ef95a is 50, key is test_row_0/A:col10/1731408889022/Put/seqid=0 2024-11-12T10:54:49,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742107_1283 (size=12301) 2024-11-12T10:54:49,989 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/1743439c7c944498ab9459c1d10ef95a 2024-11-12T10:54:50,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/69b47db8f4d0456ea1eda6d6e08a536d is 50, key is test_row_0/B:col10/1731408889022/Put/seqid=0 2024-11-12T10:54:50,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742108_1284 (size=12301) 2024-11-12T10:54:50,011 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/69b47db8f4d0456ea1eda6d6e08a536d 2024-11-12T10:54:50,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/ab1abdee625540af875e77ab0807ae26 is 50, key is test_row_0/C:col10/1731408889022/Put/seqid=0 2024-11-12T10:54:50,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742109_1285 (size=12301) 2024-11-12T10:54:50,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T10:54:50,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:50,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:50,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408950200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:50,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:50,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408950204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:50,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:50,292 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/8740ab04240844d395166a02a40197da as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8740ab04240844d395166a02a40197da 2024-11-12T10:54:50,297 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/B of 87c17057f5d8c2f7d82f49303921d612 into 8740ab04240844d395166a02a40197da(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:50,297 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:50,298 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/B, priority=13, startTime=1731408889822; duration=0sec 2024-11-12T10:54:50,298 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:50,298 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:B 2024-11-12T10:54:50,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:50,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408950307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:50,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:50,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408950316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:50,436 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/ab1abdee625540af875e77ab0807ae26 2024-11-12T10:54:50,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/1743439c7c944498ab9459c1d10ef95a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1743439c7c944498ab9459c1d10ef95a 2024-11-12T10:54:50,450 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1743439c7c944498ab9459c1d10ef95a, entries=150, sequenceid=381, filesize=12.0 K 2024-11-12T10:54:50,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/69b47db8f4d0456ea1eda6d6e08a536d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/69b47db8f4d0456ea1eda6d6e08a536d 2024-11-12T10:54:50,466 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/69b47db8f4d0456ea1eda6d6e08a536d, entries=150, sequenceid=381, filesize=12.0 K 2024-11-12T10:54:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/ab1abdee625540af875e77ab0807ae26 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/ab1abdee625540af875e77ab0807ae26 2024-11-12T10:54:50,472 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/ab1abdee625540af875e77ab0807ae26, entries=150, sequenceid=381, filesize=12.0 K 2024-11-12T10:54:50,473 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 87c17057f5d8c2f7d82f49303921d612 in 508ms, sequenceid=381, compaction requested=false 2024-11-12T10:54:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-12T10:54:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-12T10:54:50,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-12T10:54:50,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4340 sec 2024-11-12T10:54:50,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4390 sec 2024-11-12T10:54:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:50,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T10:54:50,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:50,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:50,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:50,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:50,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:50,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:50,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/ce7368f921214578a1fedb04e0835808 is 50, key is test_row_0/A:col10/1731408890199/Put/seqid=0 2024-11-12T10:54:50,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742110_1286 (size=12301) 2024-11-12T10:54:50,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408950568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:50,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:50,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408950572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:50,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408950677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:50,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408950687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:50,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408950887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:50,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408950897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:50,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/ce7368f921214578a1fedb04e0835808 2024-11-12T10:54:50,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/bd7871bac4cb42bcb70e2afe19f76cfa is 50, key is test_row_0/B:col10/1731408890199/Put/seqid=0 2024-11-12T10:54:51,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742111_1287 (size=12301) 2024-11-12T10:54:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T10:54:51,188 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:51,188 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-12T10:54:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T10:54:51,190 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:51,191 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:51,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:51,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408951197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:51,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408951207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T10:54:51,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T10:54:51,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:51,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:51,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:51,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:51,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:51,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/bd7871bac4cb42bcb70e2afe19f76cfa 2024-11-12T10:54:51,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/6bfddf257a7744729c33c5197a668873 is 50, key is test_row_0/C:col10/1731408890199/Put/seqid=0 2024-11-12T10:54:51,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742112_1288 (size=12301) 2024-11-12T10:54:51,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/6bfddf257a7744729c33c5197a668873 2024-11-12T10:54:51,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/ce7368f921214578a1fedb04e0835808 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ce7368f921214578a1fedb04e0835808 2024-11-12T10:54:51,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T10:54:51,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:51,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T10:54:51,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ce7368f921214578a1fedb04e0835808, entries=150, sequenceid=394, filesize=12.0 K 2024-11-12T10:54:51,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/bd7871bac4cb42bcb70e2afe19f76cfa as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/bd7871bac4cb42bcb70e2afe19f76cfa 2024-11-12T10:54:51,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/bd7871bac4cb42bcb70e2afe19f76cfa, entries=150, sequenceid=394, filesize=12.0 K 2024-11-12T10:54:51,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/6bfddf257a7744729c33c5197a668873 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/6bfddf257a7744729c33c5197a668873 2024-11-12T10:54:51,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/6bfddf257a7744729c33c5197a668873, entries=150, sequenceid=394, filesize=12.0 K 2024-11-12T10:54:51,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 87c17057f5d8c2f7d82f49303921d612 in 1014ms, sequenceid=394, compaction requested=true 2024-11-12T10:54:51,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:51,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:51,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:51,533 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:51,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:51,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:51,533 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:51,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:51,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:51,534 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:51,534 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/A is initiating minor compaction (all files) 2024-11-12T10:54:51,534 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/A in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:51,535 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ff3e0f5dd96643829adbc2b3cf17d889, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1743439c7c944498ab9459c1d10ef95a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ce7368f921214578a1fedb04e0835808] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=36.9 K 2024-11-12T10:54:51,535 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:51,535 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/B is initiating minor compaction (all files) 2024-11-12T10:54:51,535 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/B in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:51,535 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8740ab04240844d395166a02a40197da, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/69b47db8f4d0456ea1eda6d6e08a536d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/bd7871bac4cb42bcb70e2afe19f76cfa] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=36.9 K 2024-11-12T10:54:51,537 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 8740ab04240844d395166a02a40197da, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731408888905 2024-11-12T10:54:51,537 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff3e0f5dd96643829adbc2b3cf17d889, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731408888905 2024-11-12T10:54:51,537 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 69b47db8f4d0456ea1eda6d6e08a536d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731408889022 2024-11-12T10:54:51,538 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1743439c7c944498ab9459c1d10ef95a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731408889022 2024-11-12T10:54:51,538 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting bd7871bac4cb42bcb70e2afe19f76cfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731408890195 2024-11-12T10:54:51,538 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce7368f921214578a1fedb04e0835808, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731408890195 2024-11-12T10:54:51,550 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#B#compaction#249 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:51,551 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/0894ec9d89734980a1a1d342d6ff96ef is 50, key is test_row_0/B:col10/1731408890199/Put/seqid=0 2024-11-12T10:54:51,571 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:51,572 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/e2c67781a87d40df851409d54317dade is 50, key is test_row_0/A:col10/1731408890199/Put/seqid=0 2024-11-12T10:54:51,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742113_1289 (size=13255) 2024-11-12T10:54:51,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742114_1290 (size=13255) 2024-11-12T10:54:51,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T10:54:51,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:51,652 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T10:54:51,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:51,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:51,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:51,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:51,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:51,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:51,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/9e3716b92e284ba0b5dd4ba9fcf6c2c9 is 50, key is test_row_0/A:col10/1731408890571/Put/seqid=0 2024-11-12T10:54:51,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742115_1291 (size=12301) 2024-11-12T10:54:51,691 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/9e3716b92e284ba0b5dd4ba9fcf6c2c9 2024-11-12T10:54:51,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/27c5e04f6eaa41dbbbe3807b28ef4637 is 50, key is test_row_0/B:col10/1731408890571/Put/seqid=0 2024-11-12T10:54:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:51,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:51,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742116_1292 (size=12301) 2024-11-12T10:54:51,735 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/27c5e04f6eaa41dbbbe3807b28ef4637 2024-11-12T10:54:51,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/588d138314044f5a921902a9f3e6ee2e is 50, key is test_row_0/C:col10/1731408890571/Put/seqid=0 2024-11-12T10:54:51,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:51,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408951745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408951746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:51,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742117_1293 (size=12301) 2024-11-12T10:54:51,797 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/588d138314044f5a921902a9f3e6ee2e 2024-11-12T10:54:51,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/9e3716b92e284ba0b5dd4ba9fcf6c2c9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9e3716b92e284ba0b5dd4ba9fcf6c2c9 2024-11-12T10:54:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T10:54:51,819 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9e3716b92e284ba0b5dd4ba9fcf6c2c9, entries=150, sequenceid=414, filesize=12.0 K 2024-11-12T10:54:51,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/27c5e04f6eaa41dbbbe3807b28ef4637 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/27c5e04f6eaa41dbbbe3807b28ef4637 2024-11-12T10:54:51,827 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/27c5e04f6eaa41dbbbe3807b28ef4637, entries=150, sequenceid=414, filesize=12.0 K 2024-11-12T10:54:51,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/588d138314044f5a921902a9f3e6ee2e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/588d138314044f5a921902a9f3e6ee2e 2024-11-12T10:54:51,834 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/588d138314044f5a921902a9f3e6ee2e, entries=150, sequenceid=414, filesize=12.0 K 2024-11-12T10:54:51,835 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 87c17057f5d8c2f7d82f49303921d612 in 183ms, sequenceid=414, compaction requested=true 2024-11-12T10:54:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-12T10:54:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-12T10:54:51,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-12T10:54:51,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 645 msec 2024-11-12T10:54:51,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 651 msec 2024-11-12T10:54:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:51,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T10:54:51,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:51,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:51,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:51,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:51,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:51,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:51,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/9aab39b30d5542ccba5f9c4707b4a692 is 50, key is test_row_0/A:col10/1731408891744/Put/seqid=0 2024-11-12T10:54:51,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742118_1294 (size=12301) 2024-11-12T10:54:51,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/9aab39b30d5542ccba5f9c4707b4a692 2024-11-12T10:54:51,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/662d24b6ebe24e518751af24808ad129 is 50, key is test_row_0/B:col10/1731408891744/Put/seqid=0 2024-11-12T10:54:51,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408951933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:51,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408951940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:51,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:51,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:51,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:51,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742119_1295 (size=12301) 2024-11-12T10:54:51,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/662d24b6ebe24e518751af24808ad129 2024-11-12T10:54:51,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/d162d6ca6c25466eab86a418a102d3db is 50, key is test_row_0/C:col10/1731408891744/Put/seqid=0 2024-11-12T10:54:52,003 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/0894ec9d89734980a1a1d342d6ff96ef as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0894ec9d89734980a1a1d342d6ff96ef 2024-11-12T10:54:52,010 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/B of 87c17057f5d8c2f7d82f49303921d612 into 0894ec9d89734980a1a1d342d6ff96ef(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:52,010 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:52,010 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/B, priority=13, startTime=1731408891533; duration=0sec 2024-11-12T10:54:52,010 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:52,011 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:B 2024-11-12T10:54:52,011 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:52,012 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:52,012 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/C is initiating minor compaction (all files) 2024-11-12T10:54:52,012 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/C in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:52,012 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d28a8631929c4c2bad7d1b9ce73218e4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/ab1abdee625540af875e77ab0807ae26, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/6bfddf257a7744729c33c5197a668873, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/588d138314044f5a921902a9f3e6ee2e] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=48.9 K 2024-11-12T10:54:52,013 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting d28a8631929c4c2bad7d1b9ce73218e4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731408888905 2024-11-12T10:54:52,013 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ab1abdee625540af875e77ab0807ae26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731408889022 2024-11-12T10:54:52,014 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bfddf257a7744729c33c5197a668873, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731408890195 2024-11-12T10:54:52,016 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 588d138314044f5a921902a9f3e6ee2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731408890560 2024-11-12T10:54:52,027 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/e2c67781a87d40df851409d54317dade as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/e2c67781a87d40df851409d54317dade 2024-11-12T10:54:52,036 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/A of 87c17057f5d8c2f7d82f49303921d612 into e2c67781a87d40df851409d54317dade(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:52,036 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:52,036 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/A, priority=13, startTime=1731408891533; duration=0sec 2024-11-12T10:54:52,036 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:52,036 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:A 2024-11-12T10:54:52,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742120_1296 (size=12301) 2024-11-12T10:54:52,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/d162d6ca6c25466eab86a418a102d3db 2024-11-12T10:54:52,045 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#C#compaction#257 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:52,046 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/878afd0119bf4a61b07095455b9ca397 is 50, key is test_row_0/C:col10/1731408890571/Put/seqid=0 2024-11-12T10:54:52,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/9aab39b30d5542ccba5f9c4707b4a692 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9aab39b30d5542ccba5f9c4707b4a692 2024-11-12T10:54:52,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408952047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:52,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9aab39b30d5542ccba5f9c4707b4a692, entries=150, sequenceid=427, filesize=12.0 K 2024-11-12T10:54:52,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/662d24b6ebe24e518751af24808ad129 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/662d24b6ebe24e518751af24808ad129 2024-11-12T10:54:52,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/662d24b6ebe24e518751af24808ad129, entries=150, sequenceid=427, filesize=12.0 K 2024-11-12T10:54:52,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/d162d6ca6c25466eab86a418a102d3db as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d162d6ca6c25466eab86a418a102d3db 2024-11-12T10:54:52,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d162d6ca6c25466eab86a418a102d3db, entries=150, sequenceid=427, filesize=12.0 K 2024-11-12T10:54:52,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 87c17057f5d8c2f7d82f49303921d612 in 203ms, sequenceid=427, compaction requested=true 2024-11-12T10:54:52,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:52,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:A, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:52,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:52,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:B, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:52,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:52,071 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:52,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:52,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T10:54:52,073 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:52,073 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/A is initiating minor compaction (all files) 2024-11-12T10:54:52,073 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/A in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:52,073 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/e2c67781a87d40df851409d54317dade, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9e3716b92e284ba0b5dd4ba9fcf6c2c9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9aab39b30d5542ccba5f9c4707b4a692] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=37.0 K 2024-11-12T10:54:52,073 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2c67781a87d40df851409d54317dade, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731408890195 2024-11-12T10:54:52,074 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e3716b92e284ba0b5dd4ba9fcf6c2c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731408890560 2024-11-12T10:54:52,074 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9aab39b30d5542ccba5f9c4707b4a692, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731408891736 2024-11-12T10:54:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:52,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T10:54:52,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:52,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:52,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:52,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:52,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:52,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:52,102 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:52,103 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/5f0f2609f5144e8892f355ad204f47b7 is 50, key is test_row_0/A:col10/1731408891744/Put/seqid=0 2024-11-12T10:54:52,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742121_1297 (size=13289) 2024-11-12T10:54:52,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/f27eefcf151a45c2922cdc2c1cb787b9 is 50, key is test_row_0/A:col10/1731408891913/Put/seqid=0 2024-11-12T10:54:52,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742122_1298 (size=13357) 2024-11-12T10:54:52,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408952136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:52,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:52,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742123_1299 (size=14741) 2024-11-12T10:54:52,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408952247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:52,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:52,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408952257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T10:54:52,328 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:52,328 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-12T10:54:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T10:54:52,332 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:52,332 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:52,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T10:54:52,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408952457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:52,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:52,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T10:54:52,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:52,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:52,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:52,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:52,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:52,509 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/878afd0119bf4a61b07095455b9ca397 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/878afd0119bf4a61b07095455b9ca397 2024-11-12T10:54:52,514 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/C of 87c17057f5d8c2f7d82f49303921d612 into 878afd0119bf4a61b07095455b9ca397(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:52,514 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:52,514 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/C, priority=12, startTime=1731408891533; duration=0sec 2024-11-12T10:54:52,515 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T10:54:52,515 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:C 2024-11-12T10:54:52,515 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:C 2024-11-12T10:54:52,515 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:52,516 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:52,517 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/B is initiating minor compaction (all files) 2024-11-12T10:54:52,517 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/B in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:52,517 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0894ec9d89734980a1a1d342d6ff96ef, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/27c5e04f6eaa41dbbbe3807b28ef4637, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/662d24b6ebe24e518751af24808ad129] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=37.0 K 2024-11-12T10:54:52,519 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 0894ec9d89734980a1a1d342d6ff96ef, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731408890195 2024-11-12T10:54:52,519 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 27c5e04f6eaa41dbbbe3807b28ef4637, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731408890560 2024-11-12T10:54:52,519 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 662d24b6ebe24e518751af24808ad129, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731408891736 2024-11-12T10:54:52,529 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#B#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:52,529 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/8382a196c5c04e06b03bb52d25297a4c is 50, key is test_row_0/B:col10/1731408891744/Put/seqid=0 2024-11-12T10:54:52,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742124_1300 (size=13357) 2024-11-12T10:54:52,553 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/8382a196c5c04e06b03bb52d25297a4c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8382a196c5c04e06b03bb52d25297a4c 2024-11-12T10:54:52,554 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/5f0f2609f5144e8892f355ad204f47b7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/5f0f2609f5144e8892f355ad204f47b7 2024-11-12T10:54:52,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/f27eefcf151a45c2922cdc2c1cb787b9 2024-11-12T10:54:52,562 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/B of 87c17057f5d8c2f7d82f49303921d612 into 8382a196c5c04e06b03bb52d25297a4c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:52,562 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:52,562 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/B, priority=13, startTime=1731408892071; duration=0sec 2024-11-12T10:54:52,562 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/A of 87c17057f5d8c2f7d82f49303921d612 into 5f0f2609f5144e8892f355ad204f47b7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:52,563 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:52,563 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/A, priority=13, startTime=1731408892071; duration=0sec 2024-11-12T10:54:52,563 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:52,563 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:A 2024-11-12T10:54:52,563 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T10:54:52,565 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T10:54:52,565 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T10:54:52,565 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. because compaction request was cancelled 2024-11-12T10:54:52,565 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:C 2024-11-12T10:54:52,562 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:52,565 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:B 2024-11-12T10:54:52,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408952567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:52,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/01dde2bde5c44d96baabd53bedf76e35 is 50, key is test_row_0/B:col10/1731408891913/Put/seqid=0 2024-11-12T10:54:52,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742125_1301 (size=12301) 2024-11-12T10:54:52,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/01dde2bde5c44d96baabd53bedf76e35 2024-11-12T10:54:52,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/b2c2ee97821d48ba92e059489b701f02 is 50, key is test_row_0/C:col10/1731408891913/Put/seqid=0 2024-11-12T10:54:52,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742126_1302 (size=12301) 2024-11-12T10:54:52,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/b2c2ee97821d48ba92e059489b701f02 2024-11-12T10:54:52,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T10:54:52,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:52,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:52,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:52,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:52,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:52,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/f27eefcf151a45c2922cdc2c1cb787b9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f27eefcf151a45c2922cdc2c1cb787b9 2024-11-12T10:54:52,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f27eefcf151a45c2922cdc2c1cb787b9, entries=200, sequenceid=449, filesize=14.4 K 2024-11-12T10:54:52,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/01dde2bde5c44d96baabd53bedf76e35 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/01dde2bde5c44d96baabd53bedf76e35 2024-11-12T10:54:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T10:54:52,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/01dde2bde5c44d96baabd53bedf76e35, entries=150, sequenceid=449, filesize=12.0 K 2024-11-12T10:54:52,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/b2c2ee97821d48ba92e059489b701f02 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/b2c2ee97821d48ba92e059489b701f02 2024-11-12T10:54:52,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/b2c2ee97821d48ba92e059489b701f02, entries=150, sequenceid=449, filesize=12.0 K 2024-11-12T10:54:52,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 87c17057f5d8c2f7d82f49303921d612 in 585ms, sequenceid=449, compaction requested=true 2024-11-12T10:54:52,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:52,660 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T10:54:52,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:52,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:52,661 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T10:54:52,662 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T10:54:52,662 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T10:54:52,662 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. because compaction request was cancelled 2024-11-12T10:54:52,662 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:B 2024-11-12T10:54:52,662 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T10:54:52,662 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T10:54:52,662 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. because compaction request was cancelled 2024-11-12T10:54:52,662 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:A 2024-11-12T10:54:52,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:B, priority=-2147483648, current under compaction store size is 0 2024-11-12T10:54:52,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:52,663 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:52,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:C, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:52,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:52,664 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:52,664 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/C is initiating minor compaction (all files) 2024-11-12T10:54:52,664 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/C in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:52,665 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/878afd0119bf4a61b07095455b9ca397, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d162d6ca6c25466eab86a418a102d3db, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/b2c2ee97821d48ba92e059489b701f02] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=37.0 K 2024-11-12T10:54:52,665 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 878afd0119bf4a61b07095455b9ca397, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731408890560 2024-11-12T10:54:52,665 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting d162d6ca6c25466eab86a418a102d3db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731408891736 2024-11-12T10:54:52,665 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2c2ee97821d48ba92e059489b701f02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731408891913 2024-11-12T10:54:52,687 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:52,688 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/75bb0ea494014d8f8601f81363f61c3c is 50, key is test_row_0/C:col10/1731408891913/Put/seqid=0 2024-11-12T10:54:52,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742127_1303 (size=13391) 2024-11-12T10:54:52,724 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/75bb0ea494014d8f8601f81363f61c3c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/75bb0ea494014d8f8601f81363f61c3c 2024-11-12T10:54:52,730 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/C of 87c17057f5d8c2f7d82f49303921d612 into 75bb0ea494014d8f8601f81363f61c3c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:52,730 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:52,730 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/C, priority=13, startTime=1731408892663; duration=0sec 2024-11-12T10:54:52,730 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:52,730 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:C 2024-11-12T10:54:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:52,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T10:54:52,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:52,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:52,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:52,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:52,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:52,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:52,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/fb77910b2f854d12ab3231fc942eb270 is 50, key is test_row_0/A:col10/1731408892115/Put/seqid=0 2024-11-12T10:54:52,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742128_1304 (size=12301) 2024-11-12T10:54:52,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/fb77910b2f854d12ab3231fc942eb270 2024-11-12T10:54:52,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/eb51ea2c02674c1b9481d99618748637 is 50, key is test_row_0/B:col10/1731408892115/Put/seqid=0 2024-11-12T10:54:52,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T10:54:52,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:52,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:52,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:52,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:52,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:52,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742129_1305 (size=12301) 2024-11-12T10:54:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408952835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:52,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T10:54:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:52,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:52,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:52,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:52,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408952947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T10:54:53,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408953087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:53,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:53,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:53,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:53,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T10:54:53,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:53,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:53,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:53,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:53,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:53,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408953157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:53,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/eb51ea2c02674c1b9481d99618748637 2024-11-12T10:54:53,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/84bc08fd7ae04e67ac083388484c889e is 50, key is test_row_0/C:col10/1731408892115/Put/seqid=0 2024-11-12T10:54:53,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742130_1306 (size=12301) 2024-11-12T10:54:53,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T10:54:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:53,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:53,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T10:54:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:53,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T10:54:53,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408953467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:53,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T10:54:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:53,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:53,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/84bc08fd7ae04e67ac083388484c889e 2024-11-12T10:54:53,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/fb77910b2f854d12ab3231fc942eb270 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/fb77910b2f854d12ab3231fc942eb270 2024-11-12T10:54:53,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/fb77910b2f854d12ab3231fc942eb270, entries=150, sequenceid=466, filesize=12.0 K 2024-11-12T10:54:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/eb51ea2c02674c1b9481d99618748637 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/eb51ea2c02674c1b9481d99618748637 2024-11-12T10:54:53,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/eb51ea2c02674c1b9481d99618748637, entries=150, sequenceid=466, filesize=12.0 K 2024-11-12T10:54:53,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/84bc08fd7ae04e67ac083388484c889e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/84bc08fd7ae04e67ac083388484c889e 2024-11-12T10:54:53,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/84bc08fd7ae04e67ac083388484c889e, entries=150, sequenceid=466, filesize=12.0 K 2024-11-12T10:54:53,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 87c17057f5d8c2f7d82f49303921d612 in 863ms, sequenceid=466, compaction requested=true 2024-11-12T10:54:53,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:53,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:53,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:53,632 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:53,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:53,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:53,632 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:53,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:53,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:53,634 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:53,634 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/B is initiating minor compaction (all files) 2024-11-12T10:54:53,634 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/B in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:53,634 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8382a196c5c04e06b03bb52d25297a4c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/01dde2bde5c44d96baabd53bedf76e35, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/eb51ea2c02674c1b9481d99618748637] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=37.1 K 2024-11-12T10:54:53,635 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:53,635 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/A is initiating minor compaction (all files) 2024-11-12T10:54:53,635 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/A in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:53,635 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/5f0f2609f5144e8892f355ad204f47b7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f27eefcf151a45c2922cdc2c1cb787b9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/fb77910b2f854d12ab3231fc942eb270] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=39.5 K 2024-11-12T10:54:53,635 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 8382a196c5c04e06b03bb52d25297a4c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731408891736 2024-11-12T10:54:53,635 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f0f2609f5144e8892f355ad204f47b7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731408891736 2024-11-12T10:54:53,636 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 01dde2bde5c44d96baabd53bedf76e35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731408891913 2024-11-12T10:54:53,636 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting f27eefcf151a45c2922cdc2c1cb787b9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731408891913 2024-11-12T10:54:53,636 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb77910b2f854d12ab3231fc942eb270, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731408892115 2024-11-12T10:54:53,636 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting eb51ea2c02674c1b9481d99618748637, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731408892115 2024-11-12T10:54:53,646 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#A#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:53,647 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/f0163c0a875041e3a55c5d80ccde5ae6 is 50, key is test_row_0/A:col10/1731408892115/Put/seqid=0 2024-11-12T10:54:53,656 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#B#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:53,656 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/d26b4f5510e44972b31725e3e6c9b12a is 50, key is test_row_0/B:col10/1731408892115/Put/seqid=0 2024-11-12T10:54:53,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742131_1307 (size=13459) 2024-11-12T10:54:53,667 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/f0163c0a875041e3a55c5d80ccde5ae6 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f0163c0a875041e3a55c5d80ccde5ae6 2024-11-12T10:54:53,672 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/A of 87c17057f5d8c2f7d82f49303921d612 into f0163c0a875041e3a55c5d80ccde5ae6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:53,672 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:53,672 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/A, priority=13, startTime=1731408893632; duration=0sec 2024-11-12T10:54:53,672 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:53,672 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:A 2024-11-12T10:54:53,672 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T10:54:53,673 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T10:54:53,673 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T10:54:53,673 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. because compaction request was cancelled 2024-11-12T10:54:53,673 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:C 2024-11-12T10:54:53,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742132_1308 (size=13459) 2024-11-12T10:54:53,688 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/d26b4f5510e44972b31725e3e6c9b12a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/d26b4f5510e44972b31725e3e6c9b12a 2024-11-12T10:54:53,694 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/B of 87c17057f5d8c2f7d82f49303921d612 into d26b4f5510e44972b31725e3e6c9b12a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:53,694 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:53,694 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/B, priority=13, startTime=1731408893632; duration=0sec 2024-11-12T10:54:53,694 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:53,694 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:B 2024-11-12T10:54:53,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T10:54:53,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:53,709 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T10:54:53,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:53,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:53,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:53,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:53,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:53,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:53,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/365b965b8ffa4360a051299659683427 is 50, key is test_row_0/A:col10/1731408892821/Put/seqid=0 2024-11-12T10:54:53,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742133_1309 (size=12301) 2024-11-12T10:54:53,728 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/365b965b8ffa4360a051299659683427 2024-11-12T10:54:53,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/3ade82c3532741d499b6765bf0c87cad is 50, key is test_row_0/B:col10/1731408892821/Put/seqid=0 2024-11-12T10:54:53,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742134_1310 (size=12301) 2024-11-12T10:54:53,773 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/3ade82c3532741d499b6765bf0c87cad 2024-11-12T10:54:53,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/1bddfeabe5914a26a4d43840b9d85229 is 50, key is test_row_0/C:col10/1731408892821/Put/seqid=0 2024-11-12T10:54:53,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742135_1311 (size=12301) 2024-11-12T10:54:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:53,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:54,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408954017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:54,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43308 deadline: 1731408954106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:54,108 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in 87c17057f5d8c2f7d82f49303921d612 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:54,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408954127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:54,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:54,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:54,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:54,208 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/1bddfeabe5914a26a4d43840b9d85229 2024-11-12T10:54:54,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/365b965b8ffa4360a051299659683427 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/365b965b8ffa4360a051299659683427 2024-11-12T10:54:54,221 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/365b965b8ffa4360a051299659683427, entries=150, sequenceid=487, filesize=12.0 K 2024-11-12T10:54:54,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/3ade82c3532741d499b6765bf0c87cad as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/3ade82c3532741d499b6765bf0c87cad 2024-11-12T10:54:54,227 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/3ade82c3532741d499b6765bf0c87cad, entries=150, sequenceid=487, filesize=12.0 K 2024-11-12T10:54:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/1bddfeabe5914a26a4d43840b9d85229 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/1bddfeabe5914a26a4d43840b9d85229 2024-11-12T10:54:54,233 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/1bddfeabe5914a26a4d43840b9d85229, entries=150, sequenceid=487, filesize=12.0 K 2024-11-12T10:54:54,234 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 87c17057f5d8c2f7d82f49303921d612 in 525ms, sequenceid=487, compaction requested=true 2024-11-12T10:54:54,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:54,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:54,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-12T10:54:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-12T10:54:54,238 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-12T10:54:54,238 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9040 sec 2024-11-12T10:54:54,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.9110 sec 2024-11-12T10:54:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:54,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T10:54:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:54,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/d8883596e60f4ce6ac350410234215d9 is 50, key is test_row_0/A:col10/1731408894337/Put/seqid=0 2024-11-12T10:54:54,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742136_1312 (size=12301) 2024-11-12T10:54:54,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408954404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T10:54:54,477 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:54:54,478 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:54:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-12T10:54:54,480 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:54:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T10:54:54,481 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:54:54,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:54:54,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408954516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T10:54:54,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:54,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:54,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:54,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:54,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:54,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:54,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408954727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:54,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/d8883596e60f4ce6ac350410234215d9 2024-11-12T10:54:54,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:54,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:54,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:54,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:54,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:54,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T10:54:54,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/0f74e630532d4c52a1dc511379f4136f is 50, key is test_row_0/B:col10/1731408894337/Put/seqid=0 2024-11-12T10:54:54,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742137_1313 (size=12301) 2024-11-12T10:54:54,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:54,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408955037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:55,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:55,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:55,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T10:54:55,121 INFO [Thread-992 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-992. 2024-11-12T10:54:55,121 DEBUG [Thread-992 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:55,121 DEBUG [Thread-992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:55,121 DEBUG [Thread-992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:55,121 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:55,121 INFO [Thread-988 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-988. 2024-11-12T10:54:55,121 DEBUG [Thread-988 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:55,121 DEBUG [Thread-988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:55,121 DEBUG [Thread-988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:55,121 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:55,123 INFO [Thread-990 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-990. 2024-11-12T10:54:55,123 DEBUG [Thread-990 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:55,123 DEBUG [Thread-990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:55,123 DEBUG [Thread-990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:55,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:55,125 INFO [Thread-994 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-994. 2024-11-12T10:54:55,125 DEBUG [Thread-994 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:55,125 DEBUG [Thread-994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:55,125 DEBUG [Thread-994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:55,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:55,126 INFO [Thread-986 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-986. 2024-11-12T10:54:55,126 DEBUG [Thread-986 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:55,126 DEBUG [Thread-986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:55,126 DEBUG [Thread-986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:55,126 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:55,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/0f74e630532d4c52a1dc511379f4136f 2024-11-12T10:54:55,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/8cc404964a644c77afbe09eba8902c90 is 50, key is test_row_0/C:col10/1731408894337/Put/seqid=0 2024-11-12T10:54:55,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742138_1314 (size=12301) 2024-11-12T10:54:55,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:55,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43322 deadline: 1731408955548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:55,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:55,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:55,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:55,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:55,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:55,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43366 deadline: 1731408955597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:55,598 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 87c17057f5d8c2f7d82f49303921d612 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:55,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/8cc404964a644c77afbe09eba8902c90 2024-11-12T10:54:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T10:54:55,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/d8883596e60f4ce6ac350410234215d9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/d8883596e60f4ce6ac350410234215d9 2024-11-12T10:54:55,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/d8883596e60f4ce6ac350410234215d9, entries=150, sequenceid=501, filesize=12.0 K 2024-11-12T10:54:55,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/0f74e630532d4c52a1dc511379f4136f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0f74e630532d4c52a1dc511379f4136f 2024-11-12T10:54:55,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0f74e630532d4c52a1dc511379f4136f, entries=150, sequenceid=501, filesize=12.0 K 2024-11-12T10:54:55,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:54:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1731408955627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 2024-11-12T10:54:55,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/8cc404964a644c77afbe09eba8902c90 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/8cc404964a644c77afbe09eba8902c90 2024-11-12T10:54:55,627 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 87c17057f5d8c2f7d82f49303921d612 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:54:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:54:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:54:55,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/8cc404964a644c77afbe09eba8902c90, entries=150, sequenceid=501, filesize=12.0 K 2024-11-12T10:54:55,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 87c17057f5d8c2f7d82f49303921d612 in 1293ms, sequenceid=501, compaction requested=true 2024-11-12T10:54:55,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:55,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:54:55,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:55,631 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:55,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:54:55,631 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:54:55,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:55,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87c17057f5d8c2f7d82f49303921d612:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:54:55,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:55,632 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:55,632 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:54:55,632 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/A is initiating minor compaction (all files) 2024-11-12T10:54:55,632 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/B is initiating minor compaction (all files) 2024-11-12T10:54:55,632 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/A in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,632 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/B in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,632 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f0163c0a875041e3a55c5d80ccde5ae6, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/365b965b8ffa4360a051299659683427, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/d8883596e60f4ce6ac350410234215d9] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=37.2 K 2024-11-12T10:54:55,632 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/d26b4f5510e44972b31725e3e6c9b12a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/3ade82c3532741d499b6765bf0c87cad, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0f74e630532d4c52a1dc511379f4136f] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=37.2 K 2024-11-12T10:54:55,632 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0163c0a875041e3a55c5d80ccde5ae6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731408892115 2024-11-12T10:54:55,632 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting d26b4f5510e44972b31725e3e6c9b12a, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731408892115 2024-11-12T10:54:55,632 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ade82c3532741d499b6765bf0c87cad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731408892821 2024-11-12T10:54:55,632 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 365b965b8ffa4360a051299659683427, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731408892821 2024-11-12T10:54:55,633 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f74e630532d4c52a1dc511379f4136f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731408894005 2024-11-12T10:54:55,633 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8883596e60f4ce6ac350410234215d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731408894005 2024-11-12T10:54:55,639 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#A#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:55,639 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/9092dbbf96b04e9b900e00ce586c6890 is 50, key is test_row_0/A:col10/1731408894337/Put/seqid=0 2024-11-12T10:54:55,640 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:55,640 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/2add1dde1bf44f94861c9794d1b36aab is 50, key is test_row_0/B:col10/1731408894337/Put/seqid=0 2024-11-12T10:54:55,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742139_1315 (size=13561) 2024-11-12T10:54:55,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742140_1316 (size=13561) 2024-11-12T10:54:55,647 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/2add1dde1bf44f94861c9794d1b36aab as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/2add1dde1bf44f94861c9794d1b36aab 2024-11-12T10:54:55,651 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/B of 87c17057f5d8c2f7d82f49303921d612 into 2add1dde1bf44f94861c9794d1b36aab(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:55,651 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:55,651 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/B, priority=13, startTime=1731408895631; duration=0sec 2024-11-12T10:54:55,651 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:54:55,651 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:B 2024-11-12T10:54:55,652 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:54:55,652 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:54:55,653 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 87c17057f5d8c2f7d82f49303921d612/C is initiating minor compaction (all files) 2024-11-12T10:54:55,653 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87c17057f5d8c2f7d82f49303921d612/C in TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,653 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/75bb0ea494014d8f8601f81363f61c3c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/84bc08fd7ae04e67ac083388484c889e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/1bddfeabe5914a26a4d43840b9d85229, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/8cc404964a644c77afbe09eba8902c90] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp, totalSize=49.1 K 2024-11-12T10:54:55,653 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 75bb0ea494014d8f8601f81363f61c3c, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731408891913 2024-11-12T10:54:55,653 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 84bc08fd7ae04e67ac083388484c889e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731408892115 2024-11-12T10:54:55,653 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bddfeabe5914a26a4d43840b9d85229, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731408892821 2024-11-12T10:54:55,654 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cc404964a644c77afbe09eba8902c90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731408894005 2024-11-12T10:54:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:54:55,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T10:54:55,659 INFO [Thread-979 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-979. 2024-11-12T10:54:55,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:55,659 DEBUG [Thread-979 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:55,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:55,659 DEBUG [Thread-979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:55,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:55,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:55,659 DEBUG [Thread-979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:55,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:55,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:55,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:55,662 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87c17057f5d8c2f7d82f49303921d612#C#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:54:55,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/f3325dffaffe45f689bacb5f5666e58a is 50, key is test_row_0/A:col10/1731408894382/Put/seqid=0 2024-11-12T10:54:55,662 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/da3b24b97c204197bdfc27128d1f2b20 is 50, key is test_row_0/C:col10/1731408894337/Put/seqid=0 2024-11-12T10:54:55,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742142_1318 (size=13527) 2024-11-12T10:54:55,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742141_1317 (size=12301) 2024-11-12T10:54:55,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:55,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:55,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:55,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:55,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:56,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:56,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:56,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:56,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,050 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/9092dbbf96b04e9b900e00ce586c6890 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9092dbbf96b04e9b900e00ce586c6890 2024-11-12T10:54:56,058 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/A of 87c17057f5d8c2f7d82f49303921d612 into 9092dbbf96b04e9b900e00ce586c6890(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:56,058 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:56,058 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/A, priority=13, startTime=1731408895631; duration=0sec 2024-11-12T10:54:56,058 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:56,058 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:A 2024-11-12T10:54:56,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/f3325dffaffe45f689bacb5f5666e58a 2024-11-12T10:54:56,071 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/da3b24b97c204197bdfc27128d1f2b20 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/da3b24b97c204197bdfc27128d1f2b20 2024-11-12T10:54:56,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/bac60e6a84274d5e8d66e7ed0be5f61f is 50, key is test_row_0/B:col10/1731408894382/Put/seqid=0 2024-11-12T10:54:56,077 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87c17057f5d8c2f7d82f49303921d612/C of 87c17057f5d8c2f7d82f49303921d612 into da3b24b97c204197bdfc27128d1f2b20(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:54:56,077 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:56,077 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., storeName=87c17057f5d8c2f7d82f49303921d612/C, priority=12, startTime=1731408895631; duration=0sec 2024-11-12T10:54:56,078 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:54:56,078 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87c17057f5d8c2f7d82f49303921d612:C 2024-11-12T10:54:56,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742143_1319 (size=12301) 2024-11-12T10:54:56,130 INFO [Thread-981 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-981. 2024-11-12T10:54:56,130 DEBUG [Thread-981 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:56,130 DEBUG [Thread-981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:56,130 DEBUG [Thread-981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:56,130 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:56,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:56,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:56,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:56,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:56,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:56,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:56,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:56,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:56,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/bac60e6a84274d5e8d66e7ed0be5f61f 2024-11-12T10:54:56,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:56,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:56,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:56,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:56,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/776597371e3b4582a2f060b75757de5e is 50, key is test_row_0/C:col10/1731408894382/Put/seqid=0 2024-11-12T10:54:56,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742144_1320 (size=12301) 2024-11-12T10:54:56,569 INFO [Thread-977 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-977. 2024-11-12T10:54:56,569 DEBUG [Thread-977 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:54:56,569 DEBUG [Thread-977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:56,569 DEBUG [Thread-977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:54:56,569 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:54:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T10:54:56,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:56,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:56,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:56,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:56,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. as already flushing 2024-11-12T10:54:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:56,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:54:56,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/776597371e3b4582a2f060b75757de5e 2024-11-12T10:54:56,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/f3325dffaffe45f689bacb5f5666e58a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f3325dffaffe45f689bacb5f5666e58a 2024-11-12T10:54:56,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f3325dffaffe45f689bacb5f5666e58a, entries=150, sequenceid=521, filesize=12.0 K 2024-11-12T10:54:56,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/bac60e6a84274d5e8d66e7ed0be5f61f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/bac60e6a84274d5e8d66e7ed0be5f61f 2024-11-12T10:54:56,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/bac60e6a84274d5e8d66e7ed0be5f61f, entries=150, sequenceid=521, filesize=12.0 K 2024-11-12T10:54:56,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/776597371e3b4582a2f060b75757de5e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/776597371e3b4582a2f060b75757de5e 2024-11-12T10:54:56,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/776597371e3b4582a2f060b75757de5e, entries=150, sequenceid=521, filesize=12.0 K 2024-11-12T10:54:56,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=13.42 KB/13740 for 87c17057f5d8c2f7d82f49303921d612 in 1267ms, sequenceid=521, compaction requested=false 2024-11-12T10:54:56,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:56,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T10:54:56,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:56,949 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-12T10:54:56,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:54:56,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:56,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:54:56,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:56,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:54:56,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:54:56,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/4ee1ef3ff813411eb16570eadb8afb69 is 50, key is test_row_1/A:col10/1731408896568/Put/seqid=0 2024-11-12T10:54:56,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742145_1321 (size=9857) 2024-11-12T10:54:57,358 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/4ee1ef3ff813411eb16570eadb8afb69 2024-11-12T10:54:57,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/02ef8a5ea0a9471c847f191338468808 is 50, key is test_row_1/B:col10/1731408896568/Put/seqid=0 2024-11-12T10:54:57,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742146_1322 (size=9857) 2024-11-12T10:54:57,368 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/02ef8a5ea0a9471c847f191338468808 2024-11-12T10:54:57,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/2aa27ae1bf894b1cb9d6d69e58e0ea37 is 50, key is test_row_1/C:col10/1731408896568/Put/seqid=0 2024-11-12T10:54:57,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742147_1323 (size=9857) 2024-11-12T10:54:57,780 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/2aa27ae1bf894b1cb9d6d69e58e0ea37 2024-11-12T10:54:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/4ee1ef3ff813411eb16570eadb8afb69 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/4ee1ef3ff813411eb16570eadb8afb69 2024-11-12T10:54:57,788 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/4ee1ef3ff813411eb16570eadb8afb69, entries=100, sequenceid=528, filesize=9.6 K 2024-11-12T10:54:57,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/02ef8a5ea0a9471c847f191338468808 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/02ef8a5ea0a9471c847f191338468808 2024-11-12T10:54:57,793 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/02ef8a5ea0a9471c847f191338468808, entries=100, sequenceid=528, filesize=9.6 K 2024-11-12T10:54:57,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/2aa27ae1bf894b1cb9d6d69e58e0ea37 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/2aa27ae1bf894b1cb9d6d69e58e0ea37 2024-11-12T10:54:57,798 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/2aa27ae1bf894b1cb9d6d69e58e0ea37, entries=100, sequenceid=528, filesize=9.6 K 2024-11-12T10:54:57,799 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 87c17057f5d8c2f7d82f49303921d612 in 850ms, sequenceid=528, compaction requested=true 2024-11-12T10:54:57,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 87c17057f5d8c2f7d82f49303921d612: 2024-11-12T10:54:57,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:54:57,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-12T10:54:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-12T10:54:57,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-12T10:54:57,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3190 sec 2024-11-12T10:54:57,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 3.3240 sec 2024-11-12T10:54:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T10:54:58,647 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:01,377 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T10:55:05,651 INFO [Thread-983 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-983. 2024-11-12T10:55:05,651 DEBUG [Thread-983 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:55:05,651 DEBUG [Thread-983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:05,652 DEBUG [Thread-983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:05,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:55:05,701 INFO [Thread-975 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-975. 2024-11-12T10:55:05,701 DEBUG [Thread-975 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:55:05,701 DEBUG [Thread-975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:05,701 DEBUG [Thread-975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:05,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:55:05,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T10:55:05,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-12T10:55:05,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 187 2024-11-12T10:55:05,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-12T10:55:05,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 155 2024-11-12T10:55:05,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-12T10:55:05,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T10:55:05,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2737 2024-11-12T10:55:05,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2808 2024-11-12T10:55:05,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2743 2024-11-12T10:55:05,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2732 2024-11-12T10:55:05,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2800 2024-11-12T10:55:05,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T10:55:05,703 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T10:55:05,703 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T10:55:05,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:05,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:05,704 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:55:05,705 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T10:55:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T10:55:05,710 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408905710"}]},"ts":"1731408905710"} 2024-11-12T10:55:05,712 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T10:55:05,712 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T10:55:05,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T10:55:05,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87c17057f5d8c2f7d82f49303921d612, UNASSIGN}] 2024-11-12T10:55:05,716 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87c17057f5d8c2f7d82f49303921d612, UNASSIGN 2024-11-12T10:55:05,717 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=87c17057f5d8c2f7d82f49303921d612, regionState=CLOSING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:55:05,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87c17057f5d8c2f7d82f49303921d612, UNASSIGN because future has completed 2024-11-12T10:55:05,720 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T10:55:05,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:55:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T10:55:05,876 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:55:05,876 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T10:55:05,876 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 87c17057f5d8c2f7d82f49303921d612, disabling compactions & flushes 2024-11-12T10:55:05,876 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:55:05,876 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:55:05,877 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. after waiting 0 ms 2024-11-12T10:55:05,877 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:55:05,877 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 87c17057f5d8c2f7d82f49303921d612 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-12T10:55:05,878 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=A 2024-11-12T10:55:05,878 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:05,878 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=B 2024-11-12T10:55:05,878 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:05,878 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87c17057f5d8c2f7d82f49303921d612, store=C 2024-11-12T10:55:05,878 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:05,884 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/1fdc1cac2ea845abb89ae18b4bf7a567 is 50, key is test_row_1/A:col10/1731408905698/Put/seqid=0 2024-11-12T10:55:05,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742148_1324 (size=7415) 2024-11-12T10:55:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T10:55:06,289 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/1fdc1cac2ea845abb89ae18b4bf7a567 2024-11-12T10:55:06,301 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/67abf103f8e74420a4c6ac7fca0224c7 is 50, key is test_row_1/B:col10/1731408905698/Put/seqid=0 2024-11-12T10:55:06,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742149_1325 (size=7415) 2024-11-12T10:55:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T10:55:06,706 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/67abf103f8e74420a4c6ac7fca0224c7 2024-11-12T10:55:06,721 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/c631d925f640409e94f867cddedda89b is 50, key is test_row_1/C:col10/1731408905698/Put/seqid=0 2024-11-12T10:55:06,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742150_1326 (size=7415) 2024-11-12T10:55:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T10:55:07,125 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/c631d925f640409e94f867cddedda89b 2024-11-12T10:55:07,132 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/A/1fdc1cac2ea845abb89ae18b4bf7a567 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1fdc1cac2ea845abb89ae18b4bf7a567 2024-11-12T10:55:07,138 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1fdc1cac2ea845abb89ae18b4bf7a567, entries=50, sequenceid=533, filesize=7.2 K 2024-11-12T10:55:07,139 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/B/67abf103f8e74420a4c6ac7fca0224c7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/67abf103f8e74420a4c6ac7fca0224c7 2024-11-12T10:55:07,144 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/67abf103f8e74420a4c6ac7fca0224c7, entries=50, sequenceid=533, filesize=7.2 K 2024-11-12T10:55:07,145 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/.tmp/C/c631d925f640409e94f867cddedda89b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/c631d925f640409e94f867cddedda89b 2024-11-12T10:55:07,150 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/c631d925f640409e94f867cddedda89b, entries=50, sequenceid=533, filesize=7.2 K 2024-11-12T10:55:07,151 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 87c17057f5d8c2f7d82f49303921d612 in 1274ms, sequenceid=533, compaction requested=true 2024-11-12T10:55:07,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ca08d13a588442ec8dee2ca6bced5ebf, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/b3e7dd0dadb74e7db92de93677e2cb23, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/677d8ab18338468caf6cfc928e5e2d74, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/55b31b4ffb7a467ebf10cd5c0f8c590a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/959bef54260a45a3bf1c67bde29202cf, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/0d180246831647729b1380abdf4e8974, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/756e653768b44c8d96066ce1345e7eaf, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f69772e5bd5841f78b9b7a6322368a90, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/7d6d6a6f3eee4eeda3d9c98c2ee8cfed, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2765d42f7bad4a668cf0fdbeaecb9a70, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f800d4911c6143eba2b8c258f0f5081d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/c9f4602235704925b9fdb1622b52210c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/704907b93fed462ea78ad6a87586171a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/4665530e60444d56aac0419ca5a4eac1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/aeae5e734b384eb8a9be7b75a168b728, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/74ddd4d25e7542e38462a514cf7fbe55, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/546ae487207f4ca4964303e4d80169f9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/578a032f068c4579868a031c24309cd7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/54efb5d00a6b4c63a85b526009a60926, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2ca17df3f7494c01822710b895f51410, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/10b300c8f62044be868ca580459d5079, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2b9f78d2b5214046acd8e85686880e65, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ae2266ef3c0540d589a5cc429ee8fc84, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/0bdec34676714edd8431399782de72b8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ff3e0f5dd96643829adbc2b3cf17d889, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1b226b206f60442292b56047bd09feff, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1743439c7c944498ab9459c1d10ef95a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/e2c67781a87d40df851409d54317dade, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ce7368f921214578a1fedb04e0835808, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9e3716b92e284ba0b5dd4ba9fcf6c2c9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/5f0f2609f5144e8892f355ad204f47b7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9aab39b30d5542ccba5f9c4707b4a692, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f27eefcf151a45c2922cdc2c1cb787b9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f0163c0a875041e3a55c5d80ccde5ae6, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/fb77910b2f854d12ab3231fc942eb270, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/365b965b8ffa4360a051299659683427, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/d8883596e60f4ce6ac350410234215d9] to archive 2024-11-12T10:55:07,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:55:07,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ca08d13a588442ec8dee2ca6bced5ebf to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ca08d13a588442ec8dee2ca6bced5ebf 2024-11-12T10:55:07,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/b3e7dd0dadb74e7db92de93677e2cb23 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/b3e7dd0dadb74e7db92de93677e2cb23 2024-11-12T10:55:07,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/677d8ab18338468caf6cfc928e5e2d74 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/677d8ab18338468caf6cfc928e5e2d74 2024-11-12T10:55:07,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/55b31b4ffb7a467ebf10cd5c0f8c590a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/55b31b4ffb7a467ebf10cd5c0f8c590a 2024-11-12T10:55:07,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/959bef54260a45a3bf1c67bde29202cf to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/959bef54260a45a3bf1c67bde29202cf 2024-11-12T10:55:07,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/0d180246831647729b1380abdf4e8974 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/0d180246831647729b1380abdf4e8974 2024-11-12T10:55:07,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/756e653768b44c8d96066ce1345e7eaf to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/756e653768b44c8d96066ce1345e7eaf 2024-11-12T10:55:07,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f69772e5bd5841f78b9b7a6322368a90 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f69772e5bd5841f78b9b7a6322368a90 2024-11-12T10:55:07,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/7d6d6a6f3eee4eeda3d9c98c2ee8cfed to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/7d6d6a6f3eee4eeda3d9c98c2ee8cfed 2024-11-12T10:55:07,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2765d42f7bad4a668cf0fdbeaecb9a70 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2765d42f7bad4a668cf0fdbeaecb9a70 2024-11-12T10:55:07,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f800d4911c6143eba2b8c258f0f5081d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f800d4911c6143eba2b8c258f0f5081d 2024-11-12T10:55:07,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/c9f4602235704925b9fdb1622b52210c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/c9f4602235704925b9fdb1622b52210c 2024-11-12T10:55:07,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/704907b93fed462ea78ad6a87586171a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/704907b93fed462ea78ad6a87586171a 2024-11-12T10:55:07,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/4665530e60444d56aac0419ca5a4eac1 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/4665530e60444d56aac0419ca5a4eac1 2024-11-12T10:55:07,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/aeae5e734b384eb8a9be7b75a168b728 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/aeae5e734b384eb8a9be7b75a168b728 2024-11-12T10:55:07,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/74ddd4d25e7542e38462a514cf7fbe55 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/74ddd4d25e7542e38462a514cf7fbe55 2024-11-12T10:55:07,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/546ae487207f4ca4964303e4d80169f9 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/546ae487207f4ca4964303e4d80169f9 2024-11-12T10:55:07,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/578a032f068c4579868a031c24309cd7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/578a032f068c4579868a031c24309cd7 2024-11-12T10:55:07,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/54efb5d00a6b4c63a85b526009a60926 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/54efb5d00a6b4c63a85b526009a60926 2024-11-12T10:55:07,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2ca17df3f7494c01822710b895f51410 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2ca17df3f7494c01822710b895f51410 2024-11-12T10:55:07,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/10b300c8f62044be868ca580459d5079 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/10b300c8f62044be868ca580459d5079 2024-11-12T10:55:07,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2b9f78d2b5214046acd8e85686880e65 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/2b9f78d2b5214046acd8e85686880e65 2024-11-12T10:55:07,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ae2266ef3c0540d589a5cc429ee8fc84 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ae2266ef3c0540d589a5cc429ee8fc84 2024-11-12T10:55:07,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/0bdec34676714edd8431399782de72b8 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/0bdec34676714edd8431399782de72b8 2024-11-12T10:55:07,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ff3e0f5dd96643829adbc2b3cf17d889 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ff3e0f5dd96643829adbc2b3cf17d889 2024-11-12T10:55:07,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1b226b206f60442292b56047bd09feff to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1b226b206f60442292b56047bd09feff 2024-11-12T10:55:07,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1743439c7c944498ab9459c1d10ef95a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1743439c7c944498ab9459c1d10ef95a 2024-11-12T10:55:07,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/e2c67781a87d40df851409d54317dade to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/e2c67781a87d40df851409d54317dade 2024-11-12T10:55:07,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ce7368f921214578a1fedb04e0835808 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/ce7368f921214578a1fedb04e0835808 2024-11-12T10:55:07,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9e3716b92e284ba0b5dd4ba9fcf6c2c9 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9e3716b92e284ba0b5dd4ba9fcf6c2c9 2024-11-12T10:55:07,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/5f0f2609f5144e8892f355ad204f47b7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/5f0f2609f5144e8892f355ad204f47b7 2024-11-12T10:55:07,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9aab39b30d5542ccba5f9c4707b4a692 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9aab39b30d5542ccba5f9c4707b4a692 2024-11-12T10:55:07,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f27eefcf151a45c2922cdc2c1cb787b9 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f27eefcf151a45c2922cdc2c1cb787b9 2024-11-12T10:55:07,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f0163c0a875041e3a55c5d80ccde5ae6 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f0163c0a875041e3a55c5d80ccde5ae6 2024-11-12T10:55:07,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/fb77910b2f854d12ab3231fc942eb270 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/fb77910b2f854d12ab3231fc942eb270 2024-11-12T10:55:07,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/365b965b8ffa4360a051299659683427 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/365b965b8ffa4360a051299659683427 2024-11-12T10:55:07,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/d8883596e60f4ce6ac350410234215d9 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/d8883596e60f4ce6ac350410234215d9 2024-11-12T10:55:07,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/b5ddafa8537a4f4db61413e9bfbbf343, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/54dc52773a424f20ba2484b130ec32f5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/ba6d5eaf797b4853a6f73df25999471a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0cc5898576504ce8aad0e11e3d9925da, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/50e2406e4bb44e4185a126750ff87bb7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/2fc814d476e0416d83fbdc245a53de5f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0ab08b5b86dd42d990c6b8d8ecf0a4ba, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5e6fcb71906d4f27a7f75201fb8a2b27, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/1157a821b1314793b213400ffea65264, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/3c98e456dcdc43f98ad384ae698a295c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/86fbf31313bf4da893d2dbe2f9be7ded, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/9b066c90b6204da18edb2b14ed489468, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/d7708f3d05dd49c7a7295cea3da99667, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/6be09b33bace4997af0101cbf8d1926a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5ecb415f14b542eba414eeedcc98c196, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/219ceab687334ea19ed815302d6335bb, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8632d0403431400788c9029185c3b136, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/fbb43a005fc54f0f8a53987800fdf7f0, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/075ce11dbefd472f97651dc05e73d4ec, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5001fa94586942d5865469449e9a2c97, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/b02d7d4de3214eb2880a45ddb448bf77, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/a2c135e154d34289962915e8fbf873ad, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/753de3713b1e484d85c6fe357defa881, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/21f3ada9b46e447ebdcbcd3748456d28, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8740ab04240844d395166a02a40197da, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/84db52a7300f4101beb958c49f33bf68, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/69b47db8f4d0456ea1eda6d6e08a536d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0894ec9d89734980a1a1d342d6ff96ef, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/bd7871bac4cb42bcb70e2afe19f76cfa, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/27c5e04f6eaa41dbbbe3807b28ef4637, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8382a196c5c04e06b03bb52d25297a4c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/662d24b6ebe24e518751af24808ad129, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/01dde2bde5c44d96baabd53bedf76e35, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/d26b4f5510e44972b31725e3e6c9b12a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/eb51ea2c02674c1b9481d99618748637, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/3ade82c3532741d499b6765bf0c87cad, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0f74e630532d4c52a1dc511379f4136f] to archive 2024-11-12T10:55:07,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:55:07,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/b5ddafa8537a4f4db61413e9bfbbf343 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/b5ddafa8537a4f4db61413e9bfbbf343 2024-11-12T10:55:07,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/54dc52773a424f20ba2484b130ec32f5 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/54dc52773a424f20ba2484b130ec32f5 2024-11-12T10:55:07,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/ba6d5eaf797b4853a6f73df25999471a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/ba6d5eaf797b4853a6f73df25999471a 2024-11-12T10:55:07,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0cc5898576504ce8aad0e11e3d9925da to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0cc5898576504ce8aad0e11e3d9925da 2024-11-12T10:55:07,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/50e2406e4bb44e4185a126750ff87bb7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/50e2406e4bb44e4185a126750ff87bb7 2024-11-12T10:55:07,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/2fc814d476e0416d83fbdc245a53de5f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/2fc814d476e0416d83fbdc245a53de5f 2024-11-12T10:55:07,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0ab08b5b86dd42d990c6b8d8ecf0a4ba to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0ab08b5b86dd42d990c6b8d8ecf0a4ba 2024-11-12T10:55:07,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5e6fcb71906d4f27a7f75201fb8a2b27 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5e6fcb71906d4f27a7f75201fb8a2b27 2024-11-12T10:55:07,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/1157a821b1314793b213400ffea65264 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/1157a821b1314793b213400ffea65264 2024-11-12T10:55:07,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/3c98e456dcdc43f98ad384ae698a295c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/3c98e456dcdc43f98ad384ae698a295c 2024-11-12T10:55:07,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/86fbf31313bf4da893d2dbe2f9be7ded to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/86fbf31313bf4da893d2dbe2f9be7ded 2024-11-12T10:55:07,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/9b066c90b6204da18edb2b14ed489468 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/9b066c90b6204da18edb2b14ed489468 2024-11-12T10:55:07,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/d7708f3d05dd49c7a7295cea3da99667 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/d7708f3d05dd49c7a7295cea3da99667 2024-11-12T10:55:07,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/6be09b33bace4997af0101cbf8d1926a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/6be09b33bace4997af0101cbf8d1926a 2024-11-12T10:55:07,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5ecb415f14b542eba414eeedcc98c196 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5ecb415f14b542eba414eeedcc98c196 2024-11-12T10:55:07,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/219ceab687334ea19ed815302d6335bb to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/219ceab687334ea19ed815302d6335bb 2024-11-12T10:55:07,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8632d0403431400788c9029185c3b136 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8632d0403431400788c9029185c3b136 2024-11-12T10:55:07,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/fbb43a005fc54f0f8a53987800fdf7f0 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/fbb43a005fc54f0f8a53987800fdf7f0 2024-11-12T10:55:07,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/075ce11dbefd472f97651dc05e73d4ec to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/075ce11dbefd472f97651dc05e73d4ec 2024-11-12T10:55:07,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5001fa94586942d5865469449e9a2c97 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/5001fa94586942d5865469449e9a2c97 2024-11-12T10:55:07,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/b02d7d4de3214eb2880a45ddb448bf77 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/b02d7d4de3214eb2880a45ddb448bf77 2024-11-12T10:55:07,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/a2c135e154d34289962915e8fbf873ad to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/a2c135e154d34289962915e8fbf873ad 2024-11-12T10:55:07,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/753de3713b1e484d85c6fe357defa881 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/753de3713b1e484d85c6fe357defa881 2024-11-12T10:55:07,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/21f3ada9b46e447ebdcbcd3748456d28 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/21f3ada9b46e447ebdcbcd3748456d28 2024-11-12T10:55:07,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8740ab04240844d395166a02a40197da to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8740ab04240844d395166a02a40197da 2024-11-12T10:55:07,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/84db52a7300f4101beb958c49f33bf68 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/84db52a7300f4101beb958c49f33bf68 2024-11-12T10:55:07,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/69b47db8f4d0456ea1eda6d6e08a536d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/69b47db8f4d0456ea1eda6d6e08a536d 2024-11-12T10:55:07,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0894ec9d89734980a1a1d342d6ff96ef to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0894ec9d89734980a1a1d342d6ff96ef 2024-11-12T10:55:07,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/bd7871bac4cb42bcb70e2afe19f76cfa to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/bd7871bac4cb42bcb70e2afe19f76cfa 2024-11-12T10:55:07,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/27c5e04f6eaa41dbbbe3807b28ef4637 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/27c5e04f6eaa41dbbbe3807b28ef4637 2024-11-12T10:55:07,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8382a196c5c04e06b03bb52d25297a4c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/8382a196c5c04e06b03bb52d25297a4c 2024-11-12T10:55:07,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/662d24b6ebe24e518751af24808ad129 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/662d24b6ebe24e518751af24808ad129 2024-11-12T10:55:07,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/01dde2bde5c44d96baabd53bedf76e35 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/01dde2bde5c44d96baabd53bedf76e35 2024-11-12T10:55:07,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/d26b4f5510e44972b31725e3e6c9b12a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/d26b4f5510e44972b31725e3e6c9b12a 2024-11-12T10:55:07,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/eb51ea2c02674c1b9481d99618748637 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/eb51ea2c02674c1b9481d99618748637 2024-11-12T10:55:07,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/3ade82c3532741d499b6765bf0c87cad to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/3ade82c3532741d499b6765bf0c87cad 2024-11-12T10:55:07,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0f74e630532d4c52a1dc511379f4136f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/0f74e630532d4c52a1dc511379f4136f 2024-11-12T10:55:07,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/44a9ef81049943beac7ecc65e33b2c3d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/cad76160292348b89a961adc70e013f5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/34ac4d1748024f3f8a7db5749513928f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/b2df79da7f37453698f54d0d56d0f648, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/4f92b410fc174f078a3bd74fe6a33dc9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/e671c70fa66a4af6b99d438229c8094c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/132abf66e1c84122985da7d168f629bc, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/6cb632b99731492f9ea888d81ac39bff, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/0ee7173aa2e548d0ae3b203442f15249, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f7f0e940eec24cc3ab5e6576dd8eed51, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f2e7cd081b2f4ba7b64dc8fe3b29f9ac, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/712680c90ed54e4588d1d1a540863e7d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/7242c25ac3674028904c73ffb6af2a50, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f549a0e5412948db8448f2fd7ef692e3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/78d2f3f891f7416796784b3d6e38cb1d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/20d3cfd541d54af89631f5ab29f9ec33, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/07ff6e6cad3948399c9ab2ead4013364, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/498ca2207a2a43b4889b1d252d39d3d2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/ef98c0f363a64a08968ca835ee97ea77, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d60adf2159a341c6ad80fcde8a14059a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/0a5c6197950441ed8761a0b3f6dccdf7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/eade2b6d477a49ea9e7005dcbc258bad, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/31fc39be1adb449ca200b746235491ac, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/80c68b2c265a4243b6da816d140a2c00, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d28a8631929c4c2bad7d1b9ce73218e4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/7f512fbc4e7140dc8567cd697bf942c7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/ab1abdee625540af875e77ab0807ae26, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/6bfddf257a7744729c33c5197a668873, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/878afd0119bf4a61b07095455b9ca397, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/588d138314044f5a921902a9f3e6ee2e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d162d6ca6c25466eab86a418a102d3db, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/75bb0ea494014d8f8601f81363f61c3c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/b2c2ee97821d48ba92e059489b701f02, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/84bc08fd7ae04e67ac083388484c889e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/1bddfeabe5914a26a4d43840b9d85229, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/8cc404964a644c77afbe09eba8902c90] to archive 2024-11-12T10:55:07,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:55:07,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/44a9ef81049943beac7ecc65e33b2c3d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/44a9ef81049943beac7ecc65e33b2c3d 2024-11-12T10:55:07,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/cad76160292348b89a961adc70e013f5 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/cad76160292348b89a961adc70e013f5 2024-11-12T10:55:07,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/34ac4d1748024f3f8a7db5749513928f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/34ac4d1748024f3f8a7db5749513928f 2024-11-12T10:55:07,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/b2df79da7f37453698f54d0d56d0f648 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/b2df79da7f37453698f54d0d56d0f648 2024-11-12T10:55:07,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/4f92b410fc174f078a3bd74fe6a33dc9 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/4f92b410fc174f078a3bd74fe6a33dc9 2024-11-12T10:55:07,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/e671c70fa66a4af6b99d438229c8094c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/e671c70fa66a4af6b99d438229c8094c 2024-11-12T10:55:07,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/132abf66e1c84122985da7d168f629bc to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/132abf66e1c84122985da7d168f629bc 2024-11-12T10:55:07,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/6cb632b99731492f9ea888d81ac39bff to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/6cb632b99731492f9ea888d81ac39bff 2024-11-12T10:55:07,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/0ee7173aa2e548d0ae3b203442f15249 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/0ee7173aa2e548d0ae3b203442f15249 2024-11-12T10:55:07,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f7f0e940eec24cc3ab5e6576dd8eed51 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f7f0e940eec24cc3ab5e6576dd8eed51 2024-11-12T10:55:07,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f2e7cd081b2f4ba7b64dc8fe3b29f9ac to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f2e7cd081b2f4ba7b64dc8fe3b29f9ac 2024-11-12T10:55:07,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/712680c90ed54e4588d1d1a540863e7d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/712680c90ed54e4588d1d1a540863e7d 2024-11-12T10:55:07,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/7242c25ac3674028904c73ffb6af2a50 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/7242c25ac3674028904c73ffb6af2a50 2024-11-12T10:55:07,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f549a0e5412948db8448f2fd7ef692e3 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/f549a0e5412948db8448f2fd7ef692e3 2024-11-12T10:55:07,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/78d2f3f891f7416796784b3d6e38cb1d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/78d2f3f891f7416796784b3d6e38cb1d 2024-11-12T10:55:07,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/20d3cfd541d54af89631f5ab29f9ec33 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/20d3cfd541d54af89631f5ab29f9ec33 2024-11-12T10:55:07,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/07ff6e6cad3948399c9ab2ead4013364 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/07ff6e6cad3948399c9ab2ead4013364 2024-11-12T10:55:07,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/498ca2207a2a43b4889b1d252d39d3d2 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/498ca2207a2a43b4889b1d252d39d3d2 2024-11-12T10:55:07,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/ef98c0f363a64a08968ca835ee97ea77 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/ef98c0f363a64a08968ca835ee97ea77 2024-11-12T10:55:07,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d60adf2159a341c6ad80fcde8a14059a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d60adf2159a341c6ad80fcde8a14059a 2024-11-12T10:55:07,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/0a5c6197950441ed8761a0b3f6dccdf7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/0a5c6197950441ed8761a0b3f6dccdf7 2024-11-12T10:55:07,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/eade2b6d477a49ea9e7005dcbc258bad to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/eade2b6d477a49ea9e7005dcbc258bad 2024-11-12T10:55:07,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/31fc39be1adb449ca200b746235491ac to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/31fc39be1adb449ca200b746235491ac 2024-11-12T10:55:07,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/80c68b2c265a4243b6da816d140a2c00 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/80c68b2c265a4243b6da816d140a2c00 2024-11-12T10:55:07,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d28a8631929c4c2bad7d1b9ce73218e4 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d28a8631929c4c2bad7d1b9ce73218e4 2024-11-12T10:55:07,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/7f512fbc4e7140dc8567cd697bf942c7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/7f512fbc4e7140dc8567cd697bf942c7 2024-11-12T10:55:07,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/ab1abdee625540af875e77ab0807ae26 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/ab1abdee625540af875e77ab0807ae26 2024-11-12T10:55:07,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/6bfddf257a7744729c33c5197a668873 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/6bfddf257a7744729c33c5197a668873 2024-11-12T10:55:07,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/878afd0119bf4a61b07095455b9ca397 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/878afd0119bf4a61b07095455b9ca397 2024-11-12T10:55:07,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/588d138314044f5a921902a9f3e6ee2e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/588d138314044f5a921902a9f3e6ee2e 2024-11-12T10:55:07,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d162d6ca6c25466eab86a418a102d3db to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/d162d6ca6c25466eab86a418a102d3db 2024-11-12T10:55:07,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/75bb0ea494014d8f8601f81363f61c3c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/75bb0ea494014d8f8601f81363f61c3c 2024-11-12T10:55:07,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/b2c2ee97821d48ba92e059489b701f02 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/b2c2ee97821d48ba92e059489b701f02 2024-11-12T10:55:07,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/84bc08fd7ae04e67ac083388484c889e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/84bc08fd7ae04e67ac083388484c889e 2024-11-12T10:55:07,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/1bddfeabe5914a26a4d43840b9d85229 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/1bddfeabe5914a26a4d43840b9d85229 2024-11-12T10:55:07,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/8cc404964a644c77afbe09eba8902c90 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/8cc404964a644c77afbe09eba8902c90 2024-11-12T10:55:07,297 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/recovered.edits/536.seqid, newMaxSeqId=536, maxSeqId=1 2024-11-12T10:55:07,298 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. 2024-11-12T10:55:07,298 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 87c17057f5d8c2f7d82f49303921d612: Waiting for close lock at 1731408905876Running coprocessor pre-close hooks at 1731408905876Disabling compacts and flushes for region at 1731408905876Disabling writes for close at 1731408905877 (+1 ms)Obtaining lock to block concurrent updates at 1731408905877Preparing flush snapshotting stores in 87c17057f5d8c2f7d82f49303921d612 at 1731408905877Finished memstore snapshotting TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731408905878 (+1 ms)Flushing stores of TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612. at 1731408905880 (+2 ms)Flushing 87c17057f5d8c2f7d82f49303921d612/A: creating writer at 1731408905880Flushing 87c17057f5d8c2f7d82f49303921d612/A: appending metadata at 1731408905883 (+3 ms)Flushing 87c17057f5d8c2f7d82f49303921d612/A: closing flushed file at 1731408905883Flushing 87c17057f5d8c2f7d82f49303921d612/B: creating writer at 1731408906298 (+415 ms)Flushing 87c17057f5d8c2f7d82f49303921d612/B: appending metadata at 1731408906300 (+2 ms)Flushing 87c17057f5d8c2f7d82f49303921d612/B: closing flushed file at 1731408906300Flushing 87c17057f5d8c2f7d82f49303921d612/C: creating writer at 1731408906717 (+417 ms)Flushing 87c17057f5d8c2f7d82f49303921d612/C: appending metadata at 1731408906720 (+3 ms)Flushing 87c17057f5d8c2f7d82f49303921d612/C: closing flushed file at 1731408906720Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@aac61b7: reopening flushed file at 1731408907131 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ad79762: reopening flushed file at 1731408907138 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@101d4600: reopening flushed file at 1731408907144 (+6 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 87c17057f5d8c2f7d82f49303921d612 in 1274ms, sequenceid=533, compaction requested=true at 1731408907151 (+7 ms)Writing region close event to WAL at 1731408907294 (+143 ms)Running coprocessor post-close hooks at 1731408907298 (+4 ms)Closed at 1731408907298 2024-11-12T10:55:07,300 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:55:07,300 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=87c17057f5d8c2f7d82f49303921d612, regionState=CLOSED 2024-11-12T10:55:07,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:55:07,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-12T10:55:07,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 87c17057f5d8c2f7d82f49303921d612, server=aed202db7571,43143,1731408812834 in 1.5830 sec 2024-11-12T10:55:07,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-12T10:55:07,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87c17057f5d8c2f7d82f49303921d612, UNASSIGN in 1.5900 sec 2024-11-12T10:55:07,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-12T10:55:07,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5940 sec 2024-11-12T10:55:07,310 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408907310"}]},"ts":"1731408907310"} 2024-11-12T10:55:07,312 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T10:55:07,312 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T10:55:07,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6070 sec 2024-11-12T10:55:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T10:55:07,857 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:07,858 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T10:55:07,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:07,861 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:07,862 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-12T10:55:07,865 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:55:07,869 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/recovered.edits] 2024-11-12T10:55:07,874 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1fdc1cac2ea845abb89ae18b4bf7a567 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/1fdc1cac2ea845abb89ae18b4bf7a567 2024-11-12T10:55:07,877 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/4ee1ef3ff813411eb16570eadb8afb69 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/4ee1ef3ff813411eb16570eadb8afb69 2024-11-12T10:55:07,880 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9092dbbf96b04e9b900e00ce586c6890 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/9092dbbf96b04e9b900e00ce586c6890 2024-11-12T10:55:07,906 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f3325dffaffe45f689bacb5f5666e58a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/A/f3325dffaffe45f689bacb5f5666e58a 2024-11-12T10:55:07,912 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/02ef8a5ea0a9471c847f191338468808 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/02ef8a5ea0a9471c847f191338468808 2024-11-12T10:55:07,913 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/2add1dde1bf44f94861c9794d1b36aab to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/2add1dde1bf44f94861c9794d1b36aab 2024-11-12T10:55:07,927 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/67abf103f8e74420a4c6ac7fca0224c7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/67abf103f8e74420a4c6ac7fca0224c7 2024-11-12T10:55:07,929 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/bac60e6a84274d5e8d66e7ed0be5f61f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/B/bac60e6a84274d5e8d66e7ed0be5f61f 2024-11-12T10:55:07,939 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/2aa27ae1bf894b1cb9d6d69e58e0ea37 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/2aa27ae1bf894b1cb9d6d69e58e0ea37 2024-11-12T10:55:07,942 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/776597371e3b4582a2f060b75757de5e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/776597371e3b4582a2f060b75757de5e 2024-11-12T10:55:07,944 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/c631d925f640409e94f867cddedda89b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/c631d925f640409e94f867cddedda89b 2024-11-12T10:55:07,948 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/da3b24b97c204197bdfc27128d1f2b20 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/C/da3b24b97c204197bdfc27128d1f2b20 2024-11-12T10:55:07,959 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/recovered.edits/536.seqid to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612/recovered.edits/536.seqid 2024-11-12T10:55:07,961 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/87c17057f5d8c2f7d82f49303921d612 2024-11-12T10:55:07,961 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T10:55:07,963 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-12T10:55:07,967 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T10:55:07,970 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T10:55:07,971 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:07,971 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T10:55:07,971 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731408907971"}]},"ts":"9223372036854775807"} 2024-11-12T10:55:07,974 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T10:55:07,974 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 87c17057f5d8c2f7d82f49303921d612, NAME => 'TestAcidGuarantees,,1731408872916.87c17057f5d8c2f7d82f49303921d612.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T10:55:07,974 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T10:55:07,974 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731408907974"}]},"ts":"9223372036854775807"} 2024-11-12T10:55:07,976 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T10:55:07,977 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:07,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 119 msec 2024-11-12T10:55:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-12T10:55:08,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T10:55:08,178 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:08,185 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 251), OpenFileDescriptor=445 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=573 (was 555) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1851 (was 2138) 2024-11-12T10:55:08,194 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=573, ProcessCount=11, AvailableMemoryMB=1848 2024-11-12T10:55:08,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T10:55:08,195 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T10:55:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:08,197 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T10:55:08,197 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:08,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-12T10:55:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-12T10:55:08,198 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T10:55:08,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742151_1327 (size=1061) 2024-11-12T10:55:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-12T10:55:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-12T10:55:08,606 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98 2024-11-12T10:55:08,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742152_1328 (size=53) 2024-11-12T10:55:08,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:55:08,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e8a1716ac64a4675f49f2e2b91b01012, disabling compactions & flushes 2024-11-12T10:55:08,618 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:08,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:08,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. after waiting 0 ms 2024-11-12T10:55:08,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:08,618 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:08,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e8a1716ac64a4675f49f2e2b91b01012: Waiting for close lock at 1731408908618Disabling compacts and flushes for region at 1731408908618Disabling writes for close at 1731408908618Writing region close event to WAL at 1731408908618Closed at 1731408908618 2024-11-12T10:55:08,620 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T10:55:08,621 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731408908620"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731408908620"}]},"ts":"1731408908620"} 2024-11-12T10:55:08,623 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T10:55:08,624 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T10:55:08,624 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408908624"}]},"ts":"1731408908624"} 2024-11-12T10:55:08,626 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T10:55:08,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, ASSIGN}] 2024-11-12T10:55:08,628 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, ASSIGN 2024-11-12T10:55:08,628 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, ASSIGN; state=OFFLINE, location=aed202db7571,43143,1731408812834; forceNewPlan=false, retain=false 2024-11-12T10:55:08,779 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=e8a1716ac64a4675f49f2e2b91b01012, regionState=OPENING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:55:08,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, ASSIGN because future has completed 2024-11-12T10:55:08,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:55:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-12T10:55:08,937 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:08,937 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:55:08,938 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:08,938 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:55:08,938 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:08,938 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:08,941 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:08,943 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:55:08,943 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8a1716ac64a4675f49f2e2b91b01012 columnFamilyName A 2024-11-12T10:55:08,943 DEBUG [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:08,944 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.HStore(327): Store=e8a1716ac64a4675f49f2e2b91b01012/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:55:08,944 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:08,945 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:55:08,945 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8a1716ac64a4675f49f2e2b91b01012 columnFamilyName B 2024-11-12T10:55:08,945 DEBUG [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:08,945 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.HStore(327): Store=e8a1716ac64a4675f49f2e2b91b01012/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:55:08,945 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:08,946 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:55:08,947 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8a1716ac64a4675f49f2e2b91b01012 columnFamilyName C 2024-11-12T10:55:08,947 DEBUG [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:08,947 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.HStore(327): Store=e8a1716ac64a4675f49f2e2b91b01012/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:55:08,947 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:08,947 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:08,948 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:08,948 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:08,949 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:08,949 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:08,950 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T10:55:08,951 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:08,953 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:55:08,953 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened e8a1716ac64a4675f49f2e2b91b01012; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60466186, jitterRate=-0.09898361563682556}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T10:55:08,953 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:08,954 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for e8a1716ac64a4675f49f2e2b91b01012: Running coprocessor pre-open hook at 1731408908939Writing region info on filesystem at 1731408908939Initializing all the Stores at 1731408908940 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408908940Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408908940Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408908940Cleaning up temporary data from old regions at 1731408908949 (+9 ms)Running coprocessor post-open hooks at 1731408908953 (+4 ms)Region opened successfully at 1731408908954 (+1 ms) 2024-11-12T10:55:08,955 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., pid=89, masterSystemTime=1731408908933 2024-11-12T10:55:08,957 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:08,957 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:08,959 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=e8a1716ac64a4675f49f2e2b91b01012, regionState=OPEN, openSeqNum=2, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:55:08,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:55:08,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-12T10:55:08,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 in 180 msec 2024-11-12T10:55:08,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-12T10:55:08,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, ASSIGN in 336 msec 2024-11-12T10:55:08,967 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T10:55:08,967 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408908967"}]},"ts":"1731408908967"} 2024-11-12T10:55:08,968 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T10:55:08,969 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T10:55:08,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 774 msec 2024-11-12T10:55:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-12T10:55:09,338 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:09,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11317fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:09,341 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:09,342 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:09,344 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:09,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:09,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582d624d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:09,348 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a1e7db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:09,349 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:09,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:09,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:09,351 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T10:55:09,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is aed202db7571,45901,1731408812240 2024-11-12T10:55:09,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@635d735e 2024-11-12T10:55:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T10:55:09,352 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T10:55:09,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T10:55:09,354 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T10:55:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:09,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742153_1329 (size=1097) 2024-11-12T10:55:09,765 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T10:55:09,765 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T10:55:09,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T10:55:09,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, REOPEN/MOVE}] 2024-11-12T10:55:09,771 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, REOPEN/MOVE 2024-11-12T10:55:09,772 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=e8a1716ac64a4675f49f2e2b91b01012, regionState=CLOSING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:55:09,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, REOPEN/MOVE because future has completed 2024-11-12T10:55:09,773 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T10:55:09,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:55:09,925 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:09,925 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T10:55:09,925 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing e8a1716ac64a4675f49f2e2b91b01012, disabling compactions & flushes 2024-11-12T10:55:09,925 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:09,925 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:09,925 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. after waiting 0 ms 2024-11-12T10:55:09,926 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:09,928 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T10:55:09,929 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:09,929 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for e8a1716ac64a4675f49f2e2b91b01012: Waiting for close lock at 1731408909925Running coprocessor pre-close hooks at 1731408909925Disabling compacts and flushes for region at 1731408909925Disabling writes for close at 1731408909926 (+1 ms)Writing region close event to WAL at 1731408909926Running coprocessor post-close hooks at 1731408909929 (+3 ms)Closed at 1731408909929 2024-11-12T10:55:09,929 WARN [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: e8a1716ac64a4675f49f2e2b91b01012 to self. 2024-11-12T10:55:09,931 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:09,931 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=e8a1716ac64a4675f49f2e2b91b01012, regionState=CLOSED 2024-11-12T10:55:09,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:55:09,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-12T10:55:09,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 in 160 msec 2024-11-12T10:55:09,936 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, REOPEN/MOVE; state=CLOSED, location=aed202db7571,43143,1731408812834; forceNewPlan=false, retain=true 2024-11-12T10:55:10,086 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=e8a1716ac64a4675f49f2e2b91b01012, regionState=OPENING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,088 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, REOPEN/MOVE because future has completed 2024-11-12T10:55:10,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:55:10,244 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:10,244 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:55:10,244 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,244 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:55:10,244 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,244 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,246 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,247 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:55:10,247 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8a1716ac64a4675f49f2e2b91b01012 columnFamilyName A 2024-11-12T10:55:10,248 DEBUG [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:10,249 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.HStore(327): Store=e8a1716ac64a4675f49f2e2b91b01012/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:55:10,249 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,250 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:55:10,250 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8a1716ac64a4675f49f2e2b91b01012 columnFamilyName B 2024-11-12T10:55:10,250 DEBUG [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:10,251 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.HStore(327): Store=e8a1716ac64a4675f49f2e2b91b01012/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:55:10,251 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,251 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:55:10,252 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8a1716ac64a4675f49f2e2b91b01012 columnFamilyName C 2024-11-12T10:55:10,252 DEBUG [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:10,252 INFO [StoreOpener-e8a1716ac64a4675f49f2e2b91b01012-1 {}] regionserver.HStore(327): Store=e8a1716ac64a4675f49f2e2b91b01012/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:55:10,252 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:10,252 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,253 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,254 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,255 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,255 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,255 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T10:55:10,257 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,258 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened e8a1716ac64a4675f49f2e2b91b01012; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65058508, jitterRate=-0.030552685260772705}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T10:55:10,258 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,259 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for e8a1716ac64a4675f49f2e2b91b01012: Running coprocessor pre-open hook at 1731408910244Writing region info on filesystem at 1731408910244Initializing all the Stores at 1731408910245 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408910245Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408910245Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408910245Cleaning up temporary data from old regions at 1731408910255 (+10 ms)Running coprocessor post-open hooks at 1731408910258 (+3 ms)Region opened successfully at 1731408910259 (+1 ms) 2024-11-12T10:55:10,260 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., pid=94, masterSystemTime=1731408910241 2024-11-12T10:55:10,262 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:10,262 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:10,263 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=e8a1716ac64a4675f49f2e2b91b01012, regionState=OPEN, openSeqNum=5, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:55:10,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-12T10:55:10,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 in 176 msec 2024-11-12T10:55:10,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-12T10:55:10,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, REOPEN/MOVE in 496 msec 2024-11-12T10:55:10,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-12T10:55:10,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 501 msec 2024-11-12T10:55:10,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 918 msec 2024-11-12T10:55:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-12T10:55:10,275 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:10,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,276 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:10,276 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:10,277 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a92efe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:10,279 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eeb520f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:10,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:10,281 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5c447c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:10,284 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f2e408e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:10,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:10,286 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34c91b56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:10,288 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@646a4bcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:10,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:10,290 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25070b9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:10,292 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bada3da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,293 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:10,293 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:10,294 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d390e6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:10,296 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:10,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f270525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,297 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:10,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:10,298 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c7d6b67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:10,299 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@569c463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:10,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:10,301 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:10,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:10,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:10,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da70db3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:10,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:10,302 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7369e6ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:10,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:10,304 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:10,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:10,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ebe00e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:10,306 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14098f93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,307 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:10,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:10,308 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a829be1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:10,310 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:10,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ff11e24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a84aab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,313 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:10,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:10,314 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e7871, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:10,316 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:10,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cbcd8f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:10,318 DEBUG [Thread-1510 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:10,320 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:55:10,333 DEBUG [Thread-1516 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:10,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:10,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:10,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T10:55:10,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:10,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:10,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:10,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:10,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:10,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:55:10,349 DEBUG [Thread-1525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:10,352 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:10,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:55:10,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731408910348, max timestamp=1731408910361Num uniques 859; 2024-11-12T10:55:10,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T10:55:10,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731408910348, max timestamp=1731408910362Num uniques 859; 2024-11-12T10:55:10,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731408910348, max timestamp=1731408910361Num uniques 859; 2024-11-12T10:55:10,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T10:55:10,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T10:55:10,369 DEBUG [Thread-1519 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:10,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:55:10,385 DEBUG [Thread-1523 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:10,387 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:10,390 DEBUG [Thread-1514 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:55:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:10,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:10,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:55:10,401 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-12T10:55:10,405 DEBUG [Thread-1521 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T10:55:10,406 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:10,407 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:10,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:10,410 DEBUG [Thread-1521 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:10,410 DEBUG [Thread-1521 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:10,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:10,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:55:10,416 DEBUG [Thread-1508 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:10,418 DEBUG [Thread-1527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:10,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:10,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408970410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:10,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:55:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408970418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:55:10,425 DEBUG [Thread-1512 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:10,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:10,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:55:10,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408970421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1731408970422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39328 deadline: 1731408970430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122b7796aa1e4d4c9fa357531823e68044_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408910332/Put/seqid=0 2024-11-12T10:55:10,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742154_1330 (size=12154) 2024-11-12T10:55:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T10:55:10,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408970528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408970529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408970537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1731408970537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T10:55:10,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:10,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:10,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:10,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:10,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:10,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39328 deadline: 1731408970547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T10:55:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:10,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:10,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T10:55:10,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408970738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408970747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408970757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39328 deadline: 1731408970767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1731408970758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:10,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T10:55:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:10,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:10,883 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:10,890 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122b7796aa1e4d4c9fa357531823e68044_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122b7796aa1e4d4c9fa357531823e68044_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:10,896 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9e579ce4237844bfa3352d39d03bc90d, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:10,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9e579ce4237844bfa3352d39d03bc90d is 175, key is test_row_0/A:col10/1731408910332/Put/seqid=0 2024-11-12T10:55:10,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742155_1331 (size=30955) 2024-11-12T10:55:11,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T10:55:11,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:11,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:11,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:11,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T10:55:11,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408971069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:11,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408971077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:11,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:11,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408971088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:11,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:11,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39328 deadline: 1731408971097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:11,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1731408971097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:11,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T10:55:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:11,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,319 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9e579ce4237844bfa3352d39d03bc90d 2024-11-12T10:55:11,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T10:55:11,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:11,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:11,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:11,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/b92558606a824ca9bcfa121b2c33ce54 is 50, key is test_row_0/B:col10/1731408910332/Put/seqid=0 2024-11-12T10:55:11,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742156_1332 (size=12001) 2024-11-12T10:55:11,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T10:55:11,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:11,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:11,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:11,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T10:55:11,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408971577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:11,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408971597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:11,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408971607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:11,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39328 deadline: 1731408971617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:11,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1731408971617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:11,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T10:55:11,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:11,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:11,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:11,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/b92558606a824ca9bcfa121b2c33ce54 2024-11-12T10:55:11,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/2b10926c0cee4d449d6218ff7a85afff is 50, key is test_row_0/C:col10/1731408910332/Put/seqid=0 2024-11-12T10:55:11,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742157_1333 (size=12001) 2024-11-12T10:55:11,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/2b10926c0cee4d449d6218ff7a85afff 2024-11-12T10:55:11,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9e579ce4237844bfa3352d39d03bc90d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9e579ce4237844bfa3352d39d03bc90d 2024-11-12T10:55:11,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9e579ce4237844bfa3352d39d03bc90d, entries=150, sequenceid=14, filesize=30.2 K 2024-11-12T10:55:11,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/b92558606a824ca9bcfa121b2c33ce54 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/b92558606a824ca9bcfa121b2c33ce54 2024-11-12T10:55:11,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T10:55:11,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:11,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:11,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:11,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:11,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/b92558606a824ca9bcfa121b2c33ce54, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T10:55:11,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/2b10926c0cee4d449d6218ff7a85afff as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/2b10926c0cee4d449d6218ff7a85afff 2024-11-12T10:55:11,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/2b10926c0cee4d449d6218ff7a85afff, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T10:55:11,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for e8a1716ac64a4675f49f2e2b91b01012 in 1460ms, sequenceid=14, compaction requested=false 2024-11-12T10:55:11,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:11,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T10:55:11,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:11,943 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-12T10:55:11,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:11,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:55:11,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:11,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:55:11,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:11,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:55:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111203e6c877839d46c99047d5a37ea042c1_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408910407/Put/seqid=0 2024-11-12T10:55:11,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742158_1334 (size=12154) 2024-11-12T10:55:12,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:12,370 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111203e6c877839d46c99047d5a37ea042c1_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111203e6c877839d46c99047d5a37ea042c1_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:12,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/77bf59e672e24f39a1855feb1a8d76ab, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:12,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/77bf59e672e24f39a1855feb1a8d76ab is 175, key is test_row_0/A:col10/1731408910407/Put/seqid=0 2024-11-12T10:55:12,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742159_1335 (size=30955) 2024-11-12T10:55:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T10:55:12,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:12,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408972636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:12,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:12,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39328 deadline: 1731408972637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:12,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1731408972637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:12,651 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in e8a1716ac64a4675f49f2e2b91b01012 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:12,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:12,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408972650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:12,653 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in e8a1716ac64a4675f49f2e2b91b01012 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:12,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408972651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:12,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408972757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:12,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408972757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:12,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408972767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:12,781 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/77bf59e672e24f39a1855feb1a8d76ab 2024-11-12T10:55:12,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/152c82beca824887a13e66995905995f is 50, key is test_row_0/B:col10/1731408910407/Put/seqid=0 2024-11-12T10:55:12,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742160_1336 (size=12001) 2024-11-12T10:55:12,879 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T10:55:12,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408972967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:12,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408972981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:12,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:12,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408972987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:13,244 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/152c82beca824887a13e66995905995f 2024-11-12T10:55:13,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/93ccc68dfe7648e69312cd85c7c0ac9d is 50, key is test_row_0/C:col10/1731408910407/Put/seqid=0 2024-11-12T10:55:13,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408973289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:13,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408973289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:13,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742161_1337 (size=12001) 2024-11-12T10:55:13,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408973308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:13,711 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/93ccc68dfe7648e69312cd85c7c0ac9d 2024-11-12T10:55:13,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/77bf59e672e24f39a1855feb1a8d76ab as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/77bf59e672e24f39a1855feb1a8d76ab 2024-11-12T10:55:13,720 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/77bf59e672e24f39a1855feb1a8d76ab, entries=150, sequenceid=50, filesize=30.2 K 2024-11-12T10:55:13,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/152c82beca824887a13e66995905995f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/152c82beca824887a13e66995905995f 2024-11-12T10:55:13,726 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/152c82beca824887a13e66995905995f, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T10:55:13,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/93ccc68dfe7648e69312cd85c7c0ac9d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/93ccc68dfe7648e69312cd85c7c0ac9d 2024-11-12T10:55:13,734 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/93ccc68dfe7648e69312cd85c7c0ac9d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T10:55:13,735 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=46.96 KB/48090 for e8a1716ac64a4675f49f2e2b91b01012 in 1792ms, sequenceid=50, compaction requested=false 2024-11-12T10:55:13,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:13,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:13,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-12T10:55:13,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-12T10:55:13,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-12T10:55:13,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3300 sec 2024-11-12T10:55:13,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 3.3390 sec 2024-11-12T10:55:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:13,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T10:55:13,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:13,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:13,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:13,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:13,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:13,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:13,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127095e805e99849bb8c0d5a2fdef419f0_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408913819/Put/seqid=0 2024-11-12T10:55:13,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742162_1338 (size=14594) 2024-11-12T10:55:13,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408973904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:13,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:13,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408973906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408973907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:14,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408974029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:14,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408974029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:14,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:14,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408974029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:14,245 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:14,249 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127095e805e99849bb8c0d5a2fdef419f0_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127095e805e99849bb8c0d5a2fdef419f0_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:14,250 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/99cbd98532d14629abf05ded8f258d94, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:14,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/99cbd98532d14629abf05ded8f258d94 is 175, key is test_row_0/A:col10/1731408913819/Put/seqid=0 2024-11-12T10:55:14,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742163_1339 (size=39549) 2024-11-12T10:55:14,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408974247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:14,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408974247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:14,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408974247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T10:55:14,567 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:14,568 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-12T10:55:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T10:55:14,569 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:14,570 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:14,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:14,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408974567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:14,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408974567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:14,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408974567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:14,658 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/99cbd98532d14629abf05ded8f258d94 2024-11-12T10:55:14,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/4a64782477fd4c28b734338953e5af29 is 50, key is test_row_0/B:col10/1731408913819/Put/seqid=0 2024-11-12T10:55:14,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1731408974667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:14,670 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in e8a1716ac64a4675f49f2e2b91b01012 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T10:55:14,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39328 deadline: 1731408974668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:14,681 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in e8a1716ac64a4675f49f2e2b91b01012 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:14,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742164_1340 (size=12001) 2024-11-12T10:55:14,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/4a64782477fd4c28b734338953e5af29 2024-11-12T10:55:14,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/e9ca571ac84442f39f5ada8104a5fea4 is 50, key is test_row_0/C:col10/1731408913819/Put/seqid=0 2024-11-12T10:55:14,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T10:55:14,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:14,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:14,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:14,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:14,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:14,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742165_1341 (size=12001) 2024-11-12T10:55:14,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T10:55:14,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:14,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:14,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:14,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:14,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T10:55:15,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T10:55:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:15,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:15,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408975087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:15,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408975088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:15,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408975088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:15,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/e9ca571ac84442f39f5ada8104a5fea4 2024-11-12T10:55:15,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/99cbd98532d14629abf05ded8f258d94 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/99cbd98532d14629abf05ded8f258d94 2024-11-12T10:55:15,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/99cbd98532d14629abf05ded8f258d94, entries=200, sequenceid=62, filesize=38.6 K 2024-11-12T10:55:15,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/4a64782477fd4c28b734338953e5af29 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/4a64782477fd4c28b734338953e5af29 2024-11-12T10:55:15,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/4a64782477fd4c28b734338953e5af29, entries=150, sequenceid=62, filesize=11.7 K 2024-11-12T10:55:15,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/e9ca571ac84442f39f5ada8104a5fea4 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/e9ca571ac84442f39f5ada8104a5fea4 2024-11-12T10:55:15,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/e9ca571ac84442f39f5ada8104a5fea4, entries=150, sequenceid=62, filesize=11.7 K 2024-11-12T10:55:15,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e8a1716ac64a4675f49f2e2b91b01012 in 1351ms, sequenceid=62, compaction requested=true 2024-11-12T10:55:15,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:15,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:55:15,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:15,173 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:15,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:55:15,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:15,173 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:15,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:55:15,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:15,174 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:15,174 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:15,174 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/B is initiating minor compaction (all files) 2024-11-12T10:55:15,174 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/A is initiating minor compaction (all files) 2024-11-12T10:55:15,174 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/A in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:15,174 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/B in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:15,174 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/b92558606a824ca9bcfa121b2c33ce54, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/152c82beca824887a13e66995905995f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/4a64782477fd4c28b734338953e5af29] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=35.2 K 2024-11-12T10:55:15,174 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9e579ce4237844bfa3352d39d03bc90d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/77bf59e672e24f39a1855feb1a8d76ab, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/99cbd98532d14629abf05ded8f258d94] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=99.1 K 2024-11-12T10:55:15,175 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:15,175 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9e579ce4237844bfa3352d39d03bc90d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/77bf59e672e24f39a1855feb1a8d76ab, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/99cbd98532d14629abf05ded8f258d94] 2024-11-12T10:55:15,175 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting b92558606a824ca9bcfa121b2c33ce54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731408910331 2024-11-12T10:55:15,175 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e579ce4237844bfa3352d39d03bc90d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731408910331 2024-11-12T10:55:15,175 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 152c82beca824887a13e66995905995f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731408910407 2024-11-12T10:55:15,175 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77bf59e672e24f39a1855feb1a8d76ab, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731408910407 2024-11-12T10:55:15,175 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a64782477fd4c28b734338953e5af29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731408912646 2024-11-12T10:55:15,176 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99cbd98532d14629abf05ded8f258d94, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731408912646 2024-11-12T10:55:15,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T10:55:15,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:15,189 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T10:55:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T10:55:15,203 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#B#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:15,203 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/3943bd01d60e4595b4625fcb3765218b is 50, key is test_row_0/B:col10/1731408913819/Put/seqid=0 2024-11-12T10:55:15,218 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:15,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128b819e60ec854c60b12a6946d702fb3f_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408913903/Put/seqid=0 2024-11-12T10:55:15,249 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112236c112854414857828ed82b162a5f11_e8a1716ac64a4675f49f2e2b91b01012 store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:15,252 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112236c112854414857828ed82b162a5f11_e8a1716ac64a4675f49f2e2b91b01012, store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:15,252 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112236c112854414857828ed82b162a5f11_e8a1716ac64a4675f49f2e2b91b01012 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:15,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742166_1342 (size=12104) 2024-11-12T10:55:15,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742167_1343 (size=12154) 2024-11-12T10:55:15,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,293 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128b819e60ec854c60b12a6946d702fb3f_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128b819e60ec854c60b12a6946d702fb3f_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:15,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/f6eb9b7c8ff84b9e855ee8a76a353439, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:15,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/f6eb9b7c8ff84b9e855ee8a76a353439 is 175, key is test_row_0/A:col10/1731408913903/Put/seqid=0 2024-11-12T10:55:15,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742168_1344 (size=4469) 2024-11-12T10:55:15,303 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#A#compaction#297 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:15,304 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/d21d87b96da9424ea09626093b546df1 is 175, key is test_row_0/A:col10/1731408913819/Put/seqid=0 2024-11-12T10:55:15,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742169_1345 (size=30955) 2024-11-12T10:55:15,318 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/f6eb9b7c8ff84b9e855ee8a76a353439 2024-11-12T10:55:15,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/df07bc0674ee4d6e8dc6e8ed873cfe50 is 50, key is test_row_0/B:col10/1731408913903/Put/seqid=0 2024-11-12T10:55:15,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742170_1346 (size=31058) 2024-11-12T10:55:15,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742171_1347 (size=12001) 2024-11-12T10:55:15,362 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/df07bc0674ee4d6e8dc6e8ed873cfe50 2024-11-12T10:55:15,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/3bdbb4cb4b664da4a777129f024fca2b is 50, key is test_row_0/C:col10/1731408913903/Put/seqid=0 2024-11-12T10:55:15,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742172_1348 (size=12001) 2024-11-12T10:55:15,395 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/3bdbb4cb4b664da4a777129f024fca2b 2024-11-12T10:55:15,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/f6eb9b7c8ff84b9e855ee8a76a353439 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/f6eb9b7c8ff84b9e855ee8a76a353439 2024-11-12T10:55:15,410 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/f6eb9b7c8ff84b9e855ee8a76a353439, entries=150, sequenceid=82, filesize=30.2 K 2024-11-12T10:55:15,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/df07bc0674ee4d6e8dc6e8ed873cfe50 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/df07bc0674ee4d6e8dc6e8ed873cfe50 2024-11-12T10:55:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,414 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/df07bc0674ee4d6e8dc6e8ed873cfe50, entries=150, sequenceid=82, filesize=11.7 K 2024-11-12T10:55:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/3bdbb4cb4b664da4a777129f024fca2b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3bdbb4cb4b664da4a777129f024fca2b 2024-11-12T10:55:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,418 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3bdbb4cb4b664da4a777129f024fca2b, entries=150, sequenceid=82, filesize=11.7 K 2024-11-12T10:55:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,419 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for e8a1716ac64a4675f49f2e2b91b01012 in 230ms, sequenceid=82, compaction requested=true 2024-11-12T10:55:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:15,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:15,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-12T10:55:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-12T10:55:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-12T10:55:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 850 msec 2024-11-12T10:55:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 854 msec 2024-11-12T10:55:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,686 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/3943bd01d60e4595b4625fcb3765218b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/3943bd01d60e4595b4625fcb3765218b 2024-11-12T10:55:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,692 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/B of e8a1716ac64a4675f49f2e2b91b01012 into 3943bd01d60e4595b4625fcb3765218b(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:15,692 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,692 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/B, priority=13, startTime=1731408915173; duration=0sec 2024-11-12T10:55:15,692 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:15,692 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:B 2024-11-12T10:55:15,692 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:55:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,694 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:55:15,694 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/C is initiating minor compaction (all files) 2024-11-12T10:55:15,694 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/C in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:15,694 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/2b10926c0cee4d449d6218ff7a85afff, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/93ccc68dfe7648e69312cd85c7c0ac9d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/e9ca571ac84442f39f5ada8104a5fea4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3bdbb4cb4b664da4a777129f024fca2b] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=46.9 K 2024-11-12T10:55:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,694 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b10926c0cee4d449d6218ff7a85afff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731408910331 2024-11-12T10:55:15,695 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 93ccc68dfe7648e69312cd85c7c0ac9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731408910407 2024-11-12T10:55:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,695 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting e9ca571ac84442f39f5ada8104a5fea4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731408912646 2024-11-12T10:55:15,695 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bdbb4cb4b664da4a777129f024fca2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731408913879 2024-11-12T10:55:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T10:55:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,708 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,708 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#C#compaction#301 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:15,708 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/9ac9ba1495654531a97b09780e2a9481 is 50, key is test_row_0/C:col10/1731408913903/Put/seqid=0 2024-11-12T10:55:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-12T10:55:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T10:55:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,711 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,712 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:15,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742173_1349 (size=12139) 2024-11-12T10:55:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,744 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/d21d87b96da9424ea09626093b546df1 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/d21d87b96da9424ea09626093b546df1 2024-11-12T10:55:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,745 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/9ac9ba1495654531a97b09780e2a9481 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/9ac9ba1495654531a97b09780e2a9481 2024-11-12T10:55:15,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,750 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/C of e8a1716ac64a4675f49f2e2b91b01012 into 9ac9ba1495654531a97b09780e2a9481(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:15,750 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/A of e8a1716ac64a4675f49f2e2b91b01012 into d21d87b96da9424ea09626093b546df1(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:15,750 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:15,750 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:15,750 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/C, priority=12, startTime=1731408915173; duration=0sec 2024-11-12T10:55:15,750 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/A, priority=13, startTime=1731408915173; duration=0sec 2024-11-12T10:55:15,750 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:15,750 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:C 2024-11-12T10:55:15,750 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:15,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,750 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:A 2024-11-12T10:55:15,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T10:55:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T10:55:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:15,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:15,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-12T10:55:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-12T10:55:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-12T10:55:15,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-12T10:55:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 162 msec 2024-11-12T10:55:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T10:55:16,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,027 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:16,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:16,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-12T10:55:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T10:55:16,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,030 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:16,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,031 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:16,031 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T10:55:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T10:55:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:16,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:16,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:16,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-12T10:55:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-12T10:55:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-12T10:55:16,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-12T10:55:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 162 msec 2024-11-12T10:55:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T10:55:16,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:16,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:16,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:16,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111247569bfe0c814db8bb0120f4eeb82877_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_1/A:col10/1731408916307/Put/seqid=0 2024-11-12T10:55:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T10:55:16,350 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:16,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-12T10:55:16,354 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:16,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742174_1350 (size=12154) 2024-11-12T10:55:16,355 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T10:55:16,356 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:16,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:16,359 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111247569bfe0c814db8bb0120f4eeb82877_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111247569bfe0c814db8bb0120f4eeb82877_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:16,360 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/8f0e732799da40ea89e8af949fdc9448, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:16,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/8f0e732799da40ea89e8af949fdc9448 is 175, key is test_row_1/A:col10/1731408916307/Put/seqid=0 2024-11-12T10:55:16,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742175_1351 (size=30951) 2024-11-12T10:55:16,371 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/8f0e732799da40ea89e8af949fdc9448 2024-11-12T10:55:16,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/e0f29a2b523b4056808690ba861bc677 is 50, key is test_row_1/B:col10/1731408916307/Put/seqid=0 2024-11-12T10:55:16,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742176_1352 (size=9657) 2024-11-12T10:55:16,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/e0f29a2b523b4056808690ba861bc677 2024-11-12T10:55:16,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/3133f53f1a914904b61de251ea14aa8a is 50, key is test_row_1/C:col10/1731408916307/Put/seqid=0 2024-11-12T10:55:16,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408916393, max timestamp=1731408916410Num uniques 859; 2024-11-12T10:55:16,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408916393, max timestamp=1731408916410Num uniques 859; 2024-11-12T10:55:16,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:55:16,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:55:16,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408916393, max timestamp=1731408916410Num uniques 859; 2024-11-12T10:55:16,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:55:16,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742177_1353 (size=9657) 2024-11-12T10:55:16,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/3133f53f1a914904b61de251ea14aa8a 2024-11-12T10:55:16,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/8f0e732799da40ea89e8af949fdc9448 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/8f0e732799da40ea89e8af949fdc9448 2024-11-12T10:55:16,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/8f0e732799da40ea89e8af949fdc9448, entries=150, sequenceid=97, filesize=30.2 K 2024-11-12T10:55:16,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/e0f29a2b523b4056808690ba861bc677 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/e0f29a2b523b4056808690ba861bc677 2024-11-12T10:55:16,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/e0f29a2b523b4056808690ba861bc677, entries=100, sequenceid=97, filesize=9.4 K 2024-11-12T10:55:16,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/3133f53f1a914904b61de251ea14aa8a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3133f53f1a914904b61de251ea14aa8a 2024-11-12T10:55:16,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3133f53f1a914904b61de251ea14aa8a, entries=100, sequenceid=97, filesize=9.4 K 2024-11-12T10:55:16,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=80.51 KB/82440 for e8a1716ac64a4675f49f2e2b91b01012 in 149ms, sequenceid=97, compaction requested=true 2024-11-12T10:55:16,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:16,459 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:16,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:55:16,460 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92964 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:16,460 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/A is initiating minor compaction (all files) 2024-11-12T10:55:16,460 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/A in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:16,460 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/d21d87b96da9424ea09626093b546df1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/f6eb9b7c8ff84b9e855ee8a76a353439, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/8f0e732799da40ea89e8af949fdc9448] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=90.8 K 2024-11-12T10:55:16,460 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:16,460 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/d21d87b96da9424ea09626093b546df1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/f6eb9b7c8ff84b9e855ee8a76a353439, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/8f0e732799da40ea89e8af949fdc9448] 2024-11-12T10:55:16,460 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting d21d87b96da9424ea09626093b546df1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731408912646 2024-11-12T10:55:16,461 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6eb9b7c8ff84b9e855ee8a76a353439, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731408913879 2024-11-12T10:55:16,461 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f0e732799da40ea89e8af949fdc9448, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731408916270 2024-11-12T10:55:16,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:16,464 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:16,466 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:16,466 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/B is initiating minor compaction (all files) 2024-11-12T10:55:16,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:55:16,466 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/B in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:16,466 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/3943bd01d60e4595b4625fcb3765218b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/df07bc0674ee4d6e8dc6e8ed873cfe50, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/e0f29a2b523b4056808690ba861bc677] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=33.0 K 2024-11-12T10:55:16,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:16,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:55:16,467 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 3943bd01d60e4595b4625fcb3765218b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731408912646 2024-11-12T10:55:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T10:55:16,467 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting df07bc0674ee4d6e8dc6e8ed873cfe50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731408913879 2024-11-12T10:55:16,467 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting e0f29a2b523b4056808690ba861bc677, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731408916270 2024-11-12T10:55:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:16,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:16,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=100.63 KB heapSize=285.38 KB 2024-11-12T10:55:16,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:16,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:55:16,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:16,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:55:16,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:16,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:55:16,478 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:16,491 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#B#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:16,491 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/18af48d68b114756b934de995ca7a89f is 50, key is test_row_0/B:col10/1731408913903/Put/seqid=0 2024-11-12T10:55:16,501 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112fc54db3099d94fe2a561bd09650a400a_e8a1716ac64a4675f49f2e2b91b01012 store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:16,503 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112fc54db3099d94fe2a561bd09650a400a_e8a1716ac64a4675f49f2e2b91b01012, store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:16,504 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fc54db3099d94fe2a561bd09650a400a_e8a1716ac64a4675f49f2e2b91b01012 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:16,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112427820c37f6f4971b434bba37ae5dfd3_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408916468/Put/seqid=0 2024-11-12T10:55:16,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T10:55:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:16,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:16,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742178_1354 (size=12207) 2024-11-12T10:55:16,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742179_1355 (size=4469) 2024-11-12T10:55:16,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742180_1356 (size=14644) 2024-11-12T10:55:16,541 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:16,546 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112427820c37f6f4971b434bba37ae5dfd3_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112427820c37f6f4971b434bba37ae5dfd3_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:16,547 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/7a7582e9b16a4ad7a1912221923a0943, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:16,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/7a7582e9b16a4ad7a1912221923a0943 is 175, key is test_row_0/A:col10/1731408916468/Put/seqid=0 2024-11-12T10:55:16,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742181_1357 (size=39599) 2024-11-12T10:55:16,555 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/7a7582e9b16a4ad7a1912221923a0943 2024-11-12T10:55:16,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/17931ffc99be4c39bfda5d441718bedf is 50, key is test_row_0/B:col10/1731408916468/Put/seqid=0 2024-11-12T10:55:16,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408976554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:16,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:16,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742182_1358 (size=12051) 2024-11-12T10:55:16,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408976556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:16,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408976556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:16,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T10:55:16,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:16,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:16,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:16,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:16,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:16,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T10:55:16,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408976678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:16,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:16,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408976689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:16,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408976689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:16,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T10:55:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:16,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:16,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408976897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:16,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:16,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408976907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:16,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408976907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:16,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:16,936 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#A#compaction#305 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:16,937 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/1d35620272d0405896822f8bb487c797 is 175, key is test_row_0/A:col10/1731408913903/Put/seqid=0 2024-11-12T10:55:16,938 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/18af48d68b114756b934de995ca7a89f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/18af48d68b114756b934de995ca7a89f 2024-11-12T10:55:16,948 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/B of e8a1716ac64a4675f49f2e2b91b01012 into 18af48d68b114756b934de995ca7a89f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:16,948 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:16,948 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/B, priority=13, startTime=1731408916464; duration=0sec 2024-11-12T10:55:16,948 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:16,948 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:B 2024-11-12T10:55:16,948 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T10:55:16,949 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T10:55:16,949 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T10:55:16,949 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. because compaction request was cancelled 2024-11-12T10:55:16,949 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:C 2024-11-12T10:55:16,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742183_1359 (size=31268) 2024-11-12T10:55:16,964 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/1d35620272d0405896822f8bb487c797 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/1d35620272d0405896822f8bb487c797 2024-11-12T10:55:16,969 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/A of e8a1716ac64a4675f49f2e2b91b01012 into 1d35620272d0405896822f8bb487c797(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:16,969 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:16,969 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/A, priority=13, startTime=1731408916458; duration=0sec 2024-11-12T10:55:16,970 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:16,970 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:A 2024-11-12T10:55:16,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/17931ffc99be4c39bfda5d441718bedf 2024-11-12T10:55:16,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T10:55:16,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:16,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:16,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:16,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:16,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:16,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/7d11a64be1ad45f4bbe4e74a32e9ce3d is 50, key is test_row_0/C:col10/1731408916468/Put/seqid=0 2024-11-12T10:55:16,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742184_1360 (size=12051) 2024-11-12T10:55:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T10:55:17,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T10:55:17,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:17,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:17,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:17,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:17,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:17,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408977217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:17,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:17,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408977227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:17,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:17,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408977229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:17,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:17,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T10:55:17,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:17,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:17,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:17,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:17,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:17,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/7d11a64be1ad45f4bbe4e74a32e9ce3d 2024-11-12T10:55:17,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/7a7582e9b16a4ad7a1912221923a0943 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/7a7582e9b16a4ad7a1912221923a0943 2024-11-12T10:55:17,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/7a7582e9b16a4ad7a1912221923a0943, entries=200, sequenceid=129, filesize=38.7 K 2024-11-12T10:55:17,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/17931ffc99be4c39bfda5d441718bedf as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/17931ffc99be4c39bfda5d441718bedf 2024-11-12T10:55:17,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/17931ffc99be4c39bfda5d441718bedf, entries=150, sequenceid=129, filesize=11.8 K 2024-11-12T10:55:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/7d11a64be1ad45f4bbe4e74a32e9ce3d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/7d11a64be1ad45f4bbe4e74a32e9ce3d 2024-11-12T10:55:17,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/7d11a64be1ad45f4bbe4e74a32e9ce3d, entries=150, sequenceid=129, filesize=11.8 K 2024-11-12T10:55:17,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~285.33 KB/292176, currentSize=87.22 KB/89310 for e8a1716ac64a4675f49f2e2b91b01012 in 941ms, sequenceid=129, compaction requested=true 2024-11-12T10:55:17,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:17,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:55:17,415 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T10:55:17,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:17,416 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T10:55:17,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:55:17,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:17,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:55:17,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:17,416 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T10:55:17,416 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T10:55:17,416 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T10:55:17,416 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T10:55:17,416 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. because compaction request was cancelled 2024-11-12T10:55:17,416 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. because compaction request was cancelled 2024-11-12T10:55:17,416 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:B 2024-11-12T10:55:17,416 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:A 2024-11-12T10:55:17,416 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:17,417 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33847 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:17,417 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/C is initiating minor compaction (all files) 2024-11-12T10:55:17,417 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/C in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:17,417 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/9ac9ba1495654531a97b09780e2a9481, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3133f53f1a914904b61de251ea14aa8a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/7d11a64be1ad45f4bbe4e74a32e9ce3d] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=33.1 K 2024-11-12T10:55:17,418 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ac9ba1495654531a97b09780e2a9481, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731408913879 2024-11-12T10:55:17,418 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3133f53f1a914904b61de251ea14aa8a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731408916270 2024-11-12T10:55:17,418 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d11a64be1ad45f4bbe4e74a32e9ce3d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731408916445 2024-11-12T10:55:17,424 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#C#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:17,425 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/fdaa044f58944e5fbd17f2b6f8c28582 is 50, key is test_row_0/C:col10/1731408916468/Put/seqid=0 2024-11-12T10:55:17,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T10:55:17,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:17,439 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T10:55:17,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:17,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:17,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:17,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:17,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:17,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:17,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112338b2d5da2e642d19b7935072fc7cb51_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408916554/Put/seqid=0 2024-11-12T10:55:17,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742185_1361 (size=12291) 2024-11-12T10:55:17,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T10:55:17,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742186_1362 (size=12304) 2024-11-12T10:55:17,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:17,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408977805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:17,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408977814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:17,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408977815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:17,881 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/fdaa044f58944e5fbd17f2b6f8c28582 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/fdaa044f58944e5fbd17f2b6f8c28582 2024-11-12T10:55:17,890 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/C of e8a1716ac64a4675f49f2e2b91b01012 into fdaa044f58944e5fbd17f2b6f8c28582(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:17,890 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:17,890 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/C, priority=13, startTime=1731408917416; duration=0sec 2024-11-12T10:55:17,890 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:17,891 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:C 2024-11-12T10:55:17,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:17,932 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112338b2d5da2e642d19b7935072fc7cb51_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112338b2d5da2e642d19b7935072fc7cb51_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9f19c89678514ec9aa4570618e166438, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:17,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9f19c89678514ec9aa4570618e166438 is 175, key is test_row_0/A:col10/1731408916554/Put/seqid=0 2024-11-12T10:55:17,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408977930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:17,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408977938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:17,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408977938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:17,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742187_1363 (size=31105) 2024-11-12T10:55:18,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408978157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:18,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408978157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:18,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408978157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:18,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:18,368 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9f19c89678514ec9aa4570618e166438 2024-11-12T10:55:18,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/a2f81a49d847491ca5aba15c03e225ec is 50, key is test_row_0/B:col10/1731408916554/Put/seqid=0 2024-11-12T10:55:18,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742188_1364 (size=12151) 2024-11-12T10:55:18,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408978477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:18,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408978477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:18,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408978477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T10:55:18,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1731408978707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:18,713 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in e8a1716ac64a4675f49f2e2b91b01012 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8296 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:18,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39328 deadline: 1731408978717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:18,729 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in e8a1716ac64a4675f49f2e2b91b01012 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8303 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:18,847 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/a2f81a49d847491ca5aba15c03e225ec 2024-11-12T10:55:18,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/957c3aeb6c2643d88cfced990e092e2c is 50, key is test_row_0/C:col10/1731408916554/Put/seqid=0 2024-11-12T10:55:18,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742189_1365 (size=12151) 2024-11-12T10:55:18,894 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/957c3aeb6c2643d88cfced990e092e2c 2024-11-12T10:55:18,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9f19c89678514ec9aa4570618e166438 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9f19c89678514ec9aa4570618e166438 2024-11-12T10:55:18,915 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9f19c89678514ec9aa4570618e166438, entries=150, sequenceid=147, filesize=30.4 K 2024-11-12T10:55:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/a2f81a49d847491ca5aba15c03e225ec as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/a2f81a49d847491ca5aba15c03e225ec 2024-11-12T10:55:18,929 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/a2f81a49d847491ca5aba15c03e225ec, entries=150, sequenceid=147, filesize=11.9 K 2024-11-12T10:55:18,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/957c3aeb6c2643d88cfced990e092e2c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/957c3aeb6c2643d88cfced990e092e2c 2024-11-12T10:55:18,935 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/957c3aeb6c2643d88cfced990e092e2c, entries=150, sequenceid=147, filesize=11.9 K 2024-11-12T10:55:18,938 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for e8a1716ac64a4675f49f2e2b91b01012 in 1499ms, sequenceid=147, compaction requested=true 2024-11-12T10:55:18,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:18,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:18,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-12T10:55:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-12T10:55:18,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-12T10:55:18,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5830 sec 2024-11-12T10:55:18,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.5920 sec 2024-11-12T10:55:19,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T10:55:19,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:19,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:19,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:19,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:19,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:19,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:19,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111253129ff1a3b34031a2a747665b0fa20f_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408917776/Put/seqid=0 2024-11-12T10:55:19,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742190_1366 (size=14794) 2024-11-12T10:55:19,061 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:19,065 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111253129ff1a3b34031a2a747665b0fa20f_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111253129ff1a3b34031a2a747665b0fa20f_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:19,066 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/32f3f9ea84204911a9b8d805d9e1be6e, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:19,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/32f3f9ea84204911a9b8d805d9e1be6e is 175, key is test_row_0/A:col10/1731408917776/Put/seqid=0 2024-11-12T10:55:19,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408979063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:19,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742191_1367 (size=39749) 2024-11-12T10:55:19,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408979064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:19,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408979073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:19,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:19,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408979187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408979187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:19,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408979187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:19,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408979397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:19,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:19,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408979397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:19,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:19,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:19,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408979397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:19,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:19,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:19,479 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/32f3f9ea84204911a9b8d805d9e1be6e 2024-11-12T10:55:19,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/aee152b102f940daa0c2923b53577725 is 50, key is test_row_0/B:col10/1731408917776/Put/seqid=0 2024-11-12T10:55:19,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742192_1368 (size=12151) 2024-11-12T10:55:19,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/aee152b102f940daa0c2923b53577725 2024-11-12T10:55:19,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/20f87b8c24214837ace71f64d7d0b795 is 50, key is test_row_0/C:col10/1731408917776/Put/seqid=0 2024-11-12T10:55:19,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742193_1369 (size=12151) 2024-11-12T10:55:19,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408979717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:19,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408979717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:19,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:19,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:19,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408979717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:19,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:19,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:19,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:19,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/20f87b8c24214837ace71f64d7d0b795 2024-11-12T10:55:19,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/32f3f9ea84204911a9b8d805d9e1be6e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/32f3f9ea84204911a9b8d805d9e1be6e 2024-11-12T10:55:19,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/32f3f9ea84204911a9b8d805d9e1be6e, entries=200, sequenceid=166, filesize=38.8 K 2024-11-12T10:55:19,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/aee152b102f940daa0c2923b53577725 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/aee152b102f940daa0c2923b53577725 2024-11-12T10:55:19,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/aee152b102f940daa0c2923b53577725, entries=150, sequenceid=166, filesize=11.9 K 2024-11-12T10:55:19,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/20f87b8c24214837ace71f64d7d0b795 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/20f87b8c24214837ace71f64d7d0b795 2024-11-12T10:55:19,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/20f87b8c24214837ace71f64d7d0b795, entries=150, sequenceid=166, filesize=11.9 K 2024-11-12T10:55:19,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e8a1716ac64a4675f49f2e2b91b01012 in 951ms, sequenceid=166, compaction requested=true 2024-11-12T10:55:19,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:19,954 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:55:19,955 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141721 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:55:19,955 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/A is initiating minor compaction (all files) 2024-11-12T10:55:19,955 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/A in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:19,956 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/1d35620272d0405896822f8bb487c797, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/7a7582e9b16a4ad7a1912221923a0943, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9f19c89678514ec9aa4570618e166438, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/32f3f9ea84204911a9b8d805d9e1be6e] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=138.4 K 2024-11-12T10:55:19,956 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:19,956 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/1d35620272d0405896822f8bb487c797, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/7a7582e9b16a4ad7a1912221923a0943, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9f19c89678514ec9aa4570618e166438, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/32f3f9ea84204911a9b8d805d9e1be6e] 2024-11-12T10:55:19,956 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d35620272d0405896822f8bb487c797, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731408913903 2024-11-12T10:55:19,956 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a7582e9b16a4ad7a1912221923a0943, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731408916431 2024-11-12T10:55:19,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:55:19,957 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f19c89678514ec9aa4570618e166438, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731408916531 2024-11-12T10:55:19,957 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32f3f9ea84204911a9b8d805d9e1be6e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731408917776 2024-11-12T10:55:19,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:19,960 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:55:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:55:19,961 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48560 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:55:19,961 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/B is initiating minor compaction (all files) 2024-11-12T10:55:19,961 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/B in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:55:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:19,961 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/18af48d68b114756b934de995ca7a89f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/17931ffc99be4c39bfda5d441718bedf, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/a2f81a49d847491ca5aba15c03e225ec, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/aee152b102f940daa0c2923b53577725] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=47.4 K 2024-11-12T10:55:19,961 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 18af48d68b114756b934de995ca7a89f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731408913903 2024-11-12T10:55:19,962 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 17931ffc99be4c39bfda5d441718bedf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731408916445 2024-11-12T10:55:19,962 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f81a49d847491ca5aba15c03e225ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731408916531 2024-11-12T10:55:19,963 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting aee152b102f940daa0c2923b53577725, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731408917776 2024-11-12T10:55:19,965 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:19,967 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111202b5e89b1c0d45bf98fd4d4a593b7ba1_e8a1716ac64a4675f49f2e2b91b01012 store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:19,971 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111202b5e89b1c0d45bf98fd4d4a593b7ba1_e8a1716ac64a4675f49f2e2b91b01012, store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:19,971 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111202b5e89b1c0d45bf98fd4d4a593b7ba1_e8a1716ac64a4675f49f2e2b91b01012 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:19,984 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#B#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:19,984 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/dd6648126c5e42a884353077afc085b1 is 50, key is test_row_0/B:col10/1731408917776/Put/seqid=0 2024-11-12T10:55:19,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742194_1370 (size=4469) 2024-11-12T10:55:19,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742195_1371 (size=12493) 2024-11-12T10:55:20,001 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/dd6648126c5e42a884353077afc085b1 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/dd6648126c5e42a884353077afc085b1 2024-11-12T10:55:20,008 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/B of e8a1716ac64a4675f49f2e2b91b01012 into dd6648126c5e42a884353077afc085b1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:20,008 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:20,008 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/B, priority=12, startTime=1731408919960; duration=0sec 2024-11-12T10:55:20,008 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:20,008 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:B 2024-11-12T10:55:20,008 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:20,011 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:20,011 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/C is initiating minor compaction (all files) 2024-11-12T10:55:20,011 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/C in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:20,011 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/fdaa044f58944e5fbd17f2b6f8c28582, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/957c3aeb6c2643d88cfced990e092e2c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/20f87b8c24214837ace71f64d7d0b795] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=35.7 K 2024-11-12T10:55:20,012 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting fdaa044f58944e5fbd17f2b6f8c28582, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731408916445 2024-11-12T10:55:20,012 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 957c3aeb6c2643d88cfced990e092e2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731408916531 2024-11-12T10:55:20,013 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 20f87b8c24214837ace71f64d7d0b795, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731408917776 2024-11-12T10:55:20,032 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#C#compaction#319 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:20,033 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/530a31ad6c664bc996cea4d738ffed30 is 50, key is test_row_0/C:col10/1731408917776/Put/seqid=0 2024-11-12T10:55:20,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742196_1372 (size=12493) 2024-11-12T10:55:20,053 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/530a31ad6c664bc996cea4d738ffed30 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/530a31ad6c664bc996cea4d738ffed30 2024-11-12T10:55:20,060 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/C of e8a1716ac64a4675f49f2e2b91b01012 into 530a31ad6c664bc996cea4d738ffed30(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:20,060 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:20,060 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/C, priority=13, startTime=1731408919961; duration=0sec 2024-11-12T10:55:20,060 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:20,060 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:C 2024-11-12T10:55:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:20,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T10:55:20,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:20,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:20,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:20,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:20,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:20,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:20,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d9837928f8fc404ca5e7e8d963a82c5f_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408920239/Put/seqid=0 2024-11-12T10:55:20,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742197_1373 (size=14794) 2024-11-12T10:55:20,252 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:20,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d9837928f8fc404ca5e7e8d963a82c5f_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d9837928f8fc404ca5e7e8d963a82c5f_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:20,259 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/21bbc515c56e46118e76b9d0d4065efa, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:20,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/21bbc515c56e46118e76b9d0d4065efa is 175, key is test_row_0/A:col10/1731408920239/Put/seqid=0 2024-11-12T10:55:20,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742198_1374 (size=39749) 2024-11-12T10:55:20,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408980280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:20,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:20,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408980282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:20,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408980287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:20,389 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#A#compaction#317 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:20,389 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/ed7cf3acae8f44acb7c059305ad3f96a is 175, key is test_row_0/A:col10/1731408917776/Put/seqid=0 2024-11-12T10:55:20,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742199_1375 (size=31447) 2024-11-12T10:55:20,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:20,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408980397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408980397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:20,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408980398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:20,409 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/ed7cf3acae8f44acb7c059305ad3f96a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/ed7cf3acae8f44acb7c059305ad3f96a 2024-11-12T10:55:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:20,413 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/A of e8a1716ac64a4675f49f2e2b91b01012 into ed7cf3acae8f44acb7c059305ad3f96a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:20,413 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:20,413 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/A, priority=12, startTime=1731408919954; duration=0sec 2024-11-12T10:55:20,413 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:20,413 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:A 2024-11-12T10:55:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T10:55:20,527 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:20,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-12T10:55:20,531 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T10:55:20,532 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:20,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:20,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408980617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:20,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408980617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:20,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408980617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T10:55:20,666 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/21bbc515c56e46118e76b9d0d4065efa 2024-11-12T10:55:20,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/558c0d47cc1a4671bdfce45a6abe1b72 is 50, key is test_row_0/B:col10/1731408920239/Put/seqid=0 2024-11-12T10:55:20,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T10:55:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:20,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:20,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742200_1376 (size=12151) 2024-11-12T10:55:20,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T10:55:20,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:20,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:20,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:20,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:20,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T10:55:20,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408980928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:20,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408980928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:20,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408980928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:20,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T10:55:20,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:20,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:20,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:20,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:20,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:21,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/558c0d47cc1a4671bdfce45a6abe1b72 2024-11-12T10:55:21,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/38b952c6cba04722a6d2842d9764b64e is 50, key is test_row_0/C:col10/1731408920239/Put/seqid=0 2024-11-12T10:55:21,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742201_1377 (size=12151) 2024-11-12T10:55:21,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T10:55:21,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:21,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:21,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:21,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:21,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:21,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T10:55:21,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T10:55:21,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:21,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:21,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:21,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:21,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:21,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408981437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:21,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408981437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:21,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T10:55:21,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:21,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:21,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:21,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:21,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:21,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408981447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:21,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/38b952c6cba04722a6d2842d9764b64e 2024-11-12T10:55:21,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/21bbc515c56e46118e76b9d0d4065efa as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/21bbc515c56e46118e76b9d0d4065efa 2024-11-12T10:55:21,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/21bbc515c56e46118e76b9d0d4065efa, entries=200, sequenceid=183, filesize=38.8 K 2024-11-12T10:55:21,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/558c0d47cc1a4671bdfce45a6abe1b72 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/558c0d47cc1a4671bdfce45a6abe1b72 2024-11-12T10:55:21,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/558c0d47cc1a4671bdfce45a6abe1b72, entries=150, sequenceid=183, filesize=11.9 K 2024-11-12T10:55:21,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/38b952c6cba04722a6d2842d9764b64e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/38b952c6cba04722a6d2842d9764b64e 2024-11-12T10:55:21,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/38b952c6cba04722a6d2842d9764b64e, entries=150, sequenceid=183, filesize=11.9 K 2024-11-12T10:55:21,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e8a1716ac64a4675f49f2e2b91b01012 in 1301ms, sequenceid=183, compaction requested=false 2024-11-12T10:55:21,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:21,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T10:55:21,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:21,608 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T10:55:21,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:21,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:21,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:21,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:21,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:21,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:21,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d37ca085cc1c486ca31489b7dc7f9019_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408920252/Put/seqid=0 2024-11-12T10:55:21,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742202_1378 (size=12304) 2024-11-12T10:55:21,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:21,641 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d37ca085cc1c486ca31489b7dc7f9019_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d37ca085cc1c486ca31489b7dc7f9019_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:21,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/bfa8499c8f39408b871853233a771f96, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:21,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/bfa8499c8f39408b871853233a771f96 is 175, key is test_row_0/A:col10/1731408920252/Put/seqid=0 2024-11-12T10:55:21,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742203_1379 (size=31105) 2024-11-12T10:55:21,645 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/bfa8499c8f39408b871853233a771f96 2024-11-12T10:55:21,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/5cafe6a546764f86a0687c55409ff9b5 is 50, key is test_row_0/B:col10/1731408920252/Put/seqid=0 2024-11-12T10:55:21,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742204_1380 (size=12151) 2024-11-12T10:55:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T10:55:22,055 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/5cafe6a546764f86a0687c55409ff9b5 2024-11-12T10:55:22,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/ef337fdcb8874fd5ac8690b2391aec3f is 50, key is test_row_0/C:col10/1731408920252/Put/seqid=0 2024-11-12T10:55:22,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742205_1381 (size=12151) 2024-11-12T10:55:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:22,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:22,464 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/ef337fdcb8874fd5ac8690b2391aec3f 2024-11-12T10:55:22,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/bfa8499c8f39408b871853233a771f96 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/bfa8499c8f39408b871853233a771f96 2024-11-12T10:55:22,471 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/bfa8499c8f39408b871853233a771f96, entries=150, sequenceid=201, filesize=30.4 K 2024-11-12T10:55:22,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/5cafe6a546764f86a0687c55409ff9b5 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/5cafe6a546764f86a0687c55409ff9b5 2024-11-12T10:55:22,474 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/5cafe6a546764f86a0687c55409ff9b5, entries=150, sequenceid=201, filesize=11.9 K 2024-11-12T10:55:22,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/ef337fdcb8874fd5ac8690b2391aec3f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/ef337fdcb8874fd5ac8690b2391aec3f 2024-11-12T10:55:22,478 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/ef337fdcb8874fd5ac8690b2391aec3f, entries=150, sequenceid=201, filesize=11.9 K 2024-11-12T10:55:22,478 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=60.38 KB/61830 for e8a1716ac64a4675f49f2e2b91b01012 in 871ms, sequenceid=201, compaction requested=true 2024-11-12T10:55:22,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:22,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:22,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-12T10:55:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-12T10:55:22,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-12T10:55:22,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9470 sec 2024-11-12T10:55:22,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.9520 sec 2024-11-12T10:55:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:22,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T10:55:22,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:22,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:22,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:22,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:22,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:22,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:22,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125cbf843ef055441f85c35b6dddf9f867_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408922461/Put/seqid=0 2024-11-12T10:55:22,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742206_1382 (size=12304) 2024-11-12T10:55:22,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408982548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:22,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:22,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408982548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:22,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408982560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:22,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408982667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:22,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408982667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:22,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408982667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T10:55:22,677 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:22,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-12T10:55:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T10:55:22,679 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:22,679 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:22,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T10:55:22,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T10:55:22,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:22,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:22,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:22,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:22,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:22,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408982877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:22,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408982877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:22,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408982887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:22,895 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:22,899 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125cbf843ef055441f85c35b6dddf9f867_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125cbf843ef055441f85c35b6dddf9f867_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:22,900 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/7b2ec1f0b3fa440e86a020d7ffe2d12e, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:22,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/7b2ec1f0b3fa440e86a020d7ffe2d12e is 175, key is test_row_0/A:col10/1731408922461/Put/seqid=0 2024-11-12T10:55:22,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742207_1383 (size=31105) 2024-11-12T10:55:22,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T10:55:22,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:22,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:22,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:22,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:22,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T10:55:23,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T10:55:23,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:23,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:23,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:23,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408983187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:23,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408983197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:23,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408983197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:23,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T10:55:23,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:23,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:23,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:23,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,304 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/7b2ec1f0b3fa440e86a020d7ffe2d12e 2024-11-12T10:55:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T10:55:23,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/daa69a24ac894d8090158710ac652085 is 50, key is test_row_0/B:col10/1731408922461/Put/seqid=0 2024-11-12T10:55:23,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742208_1384 (size=12151) 2024-11-12T10:55:23,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T10:55:23,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:23,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:23,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:23,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T10:55:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:23,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408983697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:23,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/daa69a24ac894d8090158710ac652085 2024-11-12T10:55:23,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/22e99088a9914c18a4b12c4e9ad92065 is 50, key is test_row_0/C:col10/1731408922461/Put/seqid=0 2024-11-12T10:55:23,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408983717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:23,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408983717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:23,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742209_1385 (size=12151) 2024-11-12T10:55:23,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T10:55:23,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:23,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:23,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:23,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T10:55:23,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T10:55:23,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:23,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:23,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:23,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:24,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T10:55:24,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:24,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:24,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:24,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:24,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:24,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/22e99088a9914c18a4b12c4e9ad92065 2024-11-12T10:55:24,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/7b2ec1f0b3fa440e86a020d7ffe2d12e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/7b2ec1f0b3fa440e86a020d7ffe2d12e 2024-11-12T10:55:24,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/7b2ec1f0b3fa440e86a020d7ffe2d12e, entries=150, sequenceid=214, filesize=30.4 K 2024-11-12T10:55:24,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/daa69a24ac894d8090158710ac652085 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/daa69a24ac894d8090158710ac652085 2024-11-12T10:55:24,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/daa69a24ac894d8090158710ac652085, entries=150, sequenceid=214, filesize=11.9 K 2024-11-12T10:55:24,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/22e99088a9914c18a4b12c4e9ad92065 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/22e99088a9914c18a4b12c4e9ad92065 2024-11-12T10:55:24,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/22e99088a9914c18a4b12c4e9ad92065, entries=150, sequenceid=214, filesize=11.9 K 2024-11-12T10:55:24,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e8a1716ac64a4675f49f2e2b91b01012 in 1659ms, sequenceid=214, compaction requested=true 2024-11-12T10:55:24,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:24,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:55:24,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:24,142 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:55:24,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:55:24,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:24,143 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:55:24,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:55:24,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:24,143 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133406 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:55:24,143 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:55:24,143 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/A is initiating minor compaction (all files) 2024-11-12T10:55:24,143 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/B is initiating minor compaction (all files) 2024-11-12T10:55:24,143 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/B in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:24,143 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/A in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:24,144 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/dd6648126c5e42a884353077afc085b1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/558c0d47cc1a4671bdfce45a6abe1b72, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/5cafe6a546764f86a0687c55409ff9b5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/daa69a24ac894d8090158710ac652085] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=47.8 K 2024-11-12T10:55:24,144 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/ed7cf3acae8f44acb7c059305ad3f96a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/21bbc515c56e46118e76b9d0d4065efa, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/bfa8499c8f39408b871853233a771f96, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/7b2ec1f0b3fa440e86a020d7ffe2d12e] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=130.3 K 2024-11-12T10:55:24,144 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:24,144 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/ed7cf3acae8f44acb7c059305ad3f96a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/21bbc515c56e46118e76b9d0d4065efa, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/bfa8499c8f39408b871853233a771f96, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/7b2ec1f0b3fa440e86a020d7ffe2d12e] 2024-11-12T10:55:24,144 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting dd6648126c5e42a884353077afc085b1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731408917776 2024-11-12T10:55:24,144 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed7cf3acae8f44acb7c059305ad3f96a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731408917776 2024-11-12T10:55:24,144 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 558c0d47cc1a4671bdfce45a6abe1b72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731408919045 2024-11-12T10:55:24,144 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21bbc515c56e46118e76b9d0d4065efa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731408919045 2024-11-12T10:55:24,144 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cafe6a546764f86a0687c55409ff9b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731408920252 2024-11-12T10:55:24,144 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfa8499c8f39408b871853233a771f96, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731408920252 2024-11-12T10:55:24,144 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting daa69a24ac894d8090158710ac652085, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731408922461 2024-11-12T10:55:24,145 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b2ec1f0b3fa440e86a020d7ffe2d12e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731408922461 2024-11-12T10:55:24,149 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:24,151 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#B#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:24,151 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/e3dbd21526724934ba0460ff60d74cb0 is 50, key is test_row_0/B:col10/1731408922461/Put/seqid=0 2024-11-12T10:55:24,152 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411121d167464e1914658b34e9cf90db056cb_e8a1716ac64a4675f49f2e2b91b01012 store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:24,154 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411121d167464e1914658b34e9cf90db056cb_e8a1716ac64a4675f49f2e2b91b01012, store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:24,154 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121d167464e1914658b34e9cf90db056cb_e8a1716ac64a4675f49f2e2b91b01012 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:24,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742210_1386 (size=12629) 2024-11-12T10:55:24,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742211_1387 (size=4469) 2024-11-12T10:55:24,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T10:55:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:24,205 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T10:55:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:24,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ba379b88d9ce495e9bc96b06b1e85cd6_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408922520/Put/seqid=0 2024-11-12T10:55:24,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742212_1388 (size=12304) 2024-11-12T10:55:24,558 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#A#compaction#330 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:24,558 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/4060739899364b9d95cdeda35fac6690 is 175, key is test_row_0/A:col10/1731408922461/Put/seqid=0 2024-11-12T10:55:24,562 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/e3dbd21526724934ba0460ff60d74cb0 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/e3dbd21526724934ba0460ff60d74cb0 2024-11-12T10:55:24,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742213_1389 (size=31583) 2024-11-12T10:55:24,566 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/B of e8a1716ac64a4675f49f2e2b91b01012 into e3dbd21526724934ba0460ff60d74cb0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:24,566 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:24,566 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/B, priority=12, startTime=1731408924142; duration=0sec 2024-11-12T10:55:24,567 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:24,567 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:B 2024-11-12T10:55:24,567 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:55:24,567 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:55:24,567 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/C is initiating minor compaction (all files) 2024-11-12T10:55:24,568 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/C in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:24,568 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/530a31ad6c664bc996cea4d738ffed30, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/38b952c6cba04722a6d2842d9764b64e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/ef337fdcb8874fd5ac8690b2391aec3f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/22e99088a9914c18a4b12c4e9ad92065] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=47.8 K 2024-11-12T10:55:24,568 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 530a31ad6c664bc996cea4d738ffed30, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731408917776 2024-11-12T10:55:24,568 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b952c6cba04722a6d2842d9764b64e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731408919045 2024-11-12T10:55:24,568 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ef337fdcb8874fd5ac8690b2391aec3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731408920252 2024-11-12T10:55:24,569 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 22e99088a9914c18a4b12c4e9ad92065, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731408922461 2024-11-12T10:55:24,576 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#C#compaction#332 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:24,577 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/3cad5edd794243c6b6b3534575570607 is 50, key is test_row_0/C:col10/1731408922461/Put/seqid=0 2024-11-12T10:55:24,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742214_1390 (size=12629) 2024-11-12T10:55:24,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:24,617 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ba379b88d9ce495e9bc96b06b1e85cd6_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ba379b88d9ce495e9bc96b06b1e85cd6_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:24,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/f053eb344e94439594b468c409482150, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:24,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/f053eb344e94439594b468c409482150 is 175, key is test_row_0/A:col10/1731408922520/Put/seqid=0 2024-11-12T10:55:24,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742215_1391 (size=31105) 2024-11-12T10:55:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:24,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:24,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408984752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:24,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:24,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:24,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408984753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:24,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:24,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408984758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:24,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:24,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T10:55:24,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408984866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:24,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408984877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:24,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:24,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408984877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:24,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:24,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:24,968 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/4060739899364b9d95cdeda35fac6690 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/4060739899364b9d95cdeda35fac6690 2024-11-12T10:55:24,972 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/A of e8a1716ac64a4675f49f2e2b91b01012 into 4060739899364b9d95cdeda35fac6690(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:24,972 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:24,972 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/A, priority=12, startTime=1731408924142; duration=0sec 2024-11-12T10:55:24,972 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:24,972 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:A 2024-11-12T10:55:24,994 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/3cad5edd794243c6b6b3534575570607 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3cad5edd794243c6b6b3534575570607 2024-11-12T10:55:24,997 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/C of e8a1716ac64a4675f49f2e2b91b01012 into 3cad5edd794243c6b6b3534575570607(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:24,997 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:24,997 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/C, priority=12, startTime=1731408924143; duration=0sec 2024-11-12T10:55:24,997 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:24,997 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:C 2024-11-12T10:55:25,021 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/f053eb344e94439594b468c409482150 2024-11-12T10:55:25,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/003fcf0c7a1840d288c5992aba7e6ab6 is 50, key is test_row_0/B:col10/1731408922520/Put/seqid=0 2024-11-12T10:55:25,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742216_1392 (size=12151) 2024-11-12T10:55:25,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408985077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:25,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:25,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408985097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:25,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408985097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:25,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408985387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:25,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:25,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408985407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:25,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408985407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:25,430 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/003fcf0c7a1840d288c5992aba7e6ab6 2024-11-12T10:55:25,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/80da0ea8988041dd9755bebf750a92c4 is 50, key is test_row_0/C:col10/1731408922520/Put/seqid=0 2024-11-12T10:55:25,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742217_1393 (size=12151) 2024-11-12T10:55:25,840 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/80da0ea8988041dd9755bebf750a92c4 2024-11-12T10:55:25,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/f053eb344e94439594b468c409482150 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/f053eb344e94439594b468c409482150 2024-11-12T10:55:25,846 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/f053eb344e94439594b468c409482150, entries=150, sequenceid=233, filesize=30.4 K 2024-11-12T10:55:25,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/003fcf0c7a1840d288c5992aba7e6ab6 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/003fcf0c7a1840d288c5992aba7e6ab6 2024-11-12T10:55:25,849 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/003fcf0c7a1840d288c5992aba7e6ab6, entries=150, sequenceid=233, filesize=11.9 K 2024-11-12T10:55:25,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/80da0ea8988041dd9755bebf750a92c4 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/80da0ea8988041dd9755bebf750a92c4 2024-11-12T10:55:25,853 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/80da0ea8988041dd9755bebf750a92c4, entries=150, sequenceid=233, filesize=11.9 K 2024-11-12T10:55:25,853 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e8a1716ac64a4675f49f2e2b91b01012 in 1648ms, sequenceid=233, compaction requested=false 2024-11-12T10:55:25,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:25,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:25,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-12T10:55:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-12T10:55:25,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-12T10:55:25,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1760 sec 2024-11-12T10:55:25,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.1800 sec 2024-11-12T10:55:25,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:25,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T10:55:25,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:25,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:25,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:25,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:25,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:25,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:25,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d64d80fa95834a018eb90403811c78d6_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408924752/Put/seqid=0 2024-11-12T10:55:25,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742218_1394 (size=14794) 2024-11-12T10:55:25,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:25,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408985948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:25,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408985954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:25,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408985956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:26,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408986067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:26,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408986067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:26,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408986067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:26,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408986276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:26,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:26,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408986277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:26,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:26,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:26,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408986277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:26,314 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:26,317 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d64d80fa95834a018eb90403811c78d6_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d64d80fa95834a018eb90403811c78d6_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:26,318 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/8d848cf5e6aa4d0782e6a3a5b5bfda29, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:26,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/8d848cf5e6aa4d0782e6a3a5b5bfda29 is 175, key is test_row_0/A:col10/1731408924752/Put/seqid=0 2024-11-12T10:55:26,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742219_1395 (size=39749) 2024-11-12T10:55:26,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408986597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:26,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408986597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:26,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408986597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:26,722 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/8d848cf5e6aa4d0782e6a3a5b5bfda29 2024-11-12T10:55:26,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/0d61b817a07d4951b121e593922878b1 is 50, key is test_row_0/B:col10/1731408924752/Put/seqid=0 2024-11-12T10:55:26,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742220_1396 (size=12151) 2024-11-12T10:55:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T10:55:26,838 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:26,838 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-12T10:55:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T10:55:26,840 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:26,841 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:26,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T10:55:26,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T10:55:26,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:26,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:26,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:26,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:26,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:27,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408987107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:27,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408987117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:27,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408987117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:27,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/0d61b817a07d4951b121e593922878b1 2024-11-12T10:55:27,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/9819e133aec94be4b9444b00abbf6384 is 50, key is test_row_0/C:col10/1731408924752/Put/seqid=0 2024-11-12T10:55:27,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742221_1397 (size=12151) 2024-11-12T10:55:27,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T10:55:27,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:27,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:27,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:27,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/9819e133aec94be4b9444b00abbf6384 2024-11-12T10:55:27,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:27,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:27,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/8d848cf5e6aa4d0782e6a3a5b5bfda29 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/8d848cf5e6aa4d0782e6a3a5b5bfda29 2024-11-12T10:55:27,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/8d848cf5e6aa4d0782e6a3a5b5bfda29, entries=200, sequenceid=250, filesize=38.8 K 2024-11-12T10:55:27,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/0d61b817a07d4951b121e593922878b1 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/0d61b817a07d4951b121e593922878b1 2024-11-12T10:55:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T10:55:27,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/0d61b817a07d4951b121e593922878b1, entries=150, sequenceid=250, filesize=11.9 K 2024-11-12T10:55:27,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/9819e133aec94be4b9444b00abbf6384 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/9819e133aec94be4b9444b00abbf6384 2024-11-12T10:55:27,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/9819e133aec94be4b9444b00abbf6384, entries=150, sequenceid=250, filesize=11.9 K 2024-11-12T10:55:27,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e8a1716ac64a4675f49f2e2b91b01012 in 1262ms, sequenceid=250, compaction requested=true 2024-11-12T10:55:27,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:27,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:55:27,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:27,165 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:27,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:55:27,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:27,165 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:27,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:55:27,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:27,166 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:27,166 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:27,166 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/B is initiating minor compaction (all files) 2024-11-12T10:55:27,166 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/A is initiating minor compaction (all files) 2024-11-12T10:55:27,166 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/B in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:27,166 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/A in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:27,166 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/e3dbd21526724934ba0460ff60d74cb0, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/003fcf0c7a1840d288c5992aba7e6ab6, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/0d61b817a07d4951b121e593922878b1] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=36.1 K 2024-11-12T10:55:27,166 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/4060739899364b9d95cdeda35fac6690, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/f053eb344e94439594b468c409482150, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/8d848cf5e6aa4d0782e6a3a5b5bfda29] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=100.0 K 2024-11-12T10:55:27,166 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:27,166 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/4060739899364b9d95cdeda35fac6690, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/f053eb344e94439594b468c409482150, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/8d848cf5e6aa4d0782e6a3a5b5bfda29] 2024-11-12T10:55:27,167 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting e3dbd21526724934ba0460ff60d74cb0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731408922461 2024-11-12T10:55:27,167 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4060739899364b9d95cdeda35fac6690, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731408922461 2024-11-12T10:55:27,167 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 003fcf0c7a1840d288c5992aba7e6ab6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731408922505 2024-11-12T10:55:27,167 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting f053eb344e94439594b468c409482150, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731408922505 2024-11-12T10:55:27,167 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d61b817a07d4951b121e593922878b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731408924752 2024-11-12T10:55:27,167 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d848cf5e6aa4d0782e6a3a5b5bfda29, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731408924751 2024-11-12T10:55:27,181 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#B#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:27,181 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:27,181 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/5c40d80eb1a2498fb884fd3eb62921c3 is 50, key is test_row_0/B:col10/1731408924752/Put/seqid=0 2024-11-12T10:55:27,183 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112520c3d680e1942afb65e13ba6d0acf4c_e8a1716ac64a4675f49f2e2b91b01012 store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:27,185 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112520c3d680e1942afb65e13ba6d0acf4c_e8a1716ac64a4675f49f2e2b91b01012, store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:27,185 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112520c3d680e1942afb65e13ba6d0acf4c_e8a1716ac64a4675f49f2e2b91b01012 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:27,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742223_1399 (size=4469) 2024-11-12T10:55:27,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742222_1398 (size=12731) 2024-11-12T10:55:27,208 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/5c40d80eb1a2498fb884fd3eb62921c3 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/5c40d80eb1a2498fb884fd3eb62921c3 2024-11-12T10:55:27,214 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/B of e8a1716ac64a4675f49f2e2b91b01012 into 5c40d80eb1a2498fb884fd3eb62921c3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:27,214 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:27,214 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/B, priority=13, startTime=1731408927165; duration=0sec 2024-11-12T10:55:27,214 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:27,214 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:B 2024-11-12T10:55:27,214 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:27,218 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:27,218 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/C is initiating minor compaction (all files) 2024-11-12T10:55:27,218 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/C in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:27,218 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3cad5edd794243c6b6b3534575570607, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/80da0ea8988041dd9755bebf750a92c4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/9819e133aec94be4b9444b00abbf6384] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=36.1 K 2024-11-12T10:55:27,218 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cad5edd794243c6b6b3534575570607, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731408922461 2024-11-12T10:55:27,221 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 80da0ea8988041dd9755bebf750a92c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731408922505 2024-11-12T10:55:27,221 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 9819e133aec94be4b9444b00abbf6384, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731408924752 2024-11-12T10:55:27,230 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#C#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:27,230 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/509b0c24ec754b0f924809d00154ebd7 is 50, key is test_row_0/C:col10/1731408924752/Put/seqid=0 2024-11-12T10:55:27,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742224_1400 (size=12731) 2024-11-12T10:55:27,248 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/509b0c24ec754b0f924809d00154ebd7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/509b0c24ec754b0f924809d00154ebd7 2024-11-12T10:55:27,255 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/C of e8a1716ac64a4675f49f2e2b91b01012 into 509b0c24ec754b0f924809d00154ebd7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:27,255 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:27,255 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/C, priority=13, startTime=1731408927165; duration=0sec 2024-11-12T10:55:27,255 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:27,255 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:C 2024-11-12T10:55:27,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T10:55:27,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:27,302 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T10:55:27,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:27,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:27,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:27,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:27,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:27,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:27,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111254f855cbc0d24dffa701d25ddf02b909_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408925944/Put/seqid=0 2024-11-12T10:55:27,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742225_1401 (size=12454) 2024-11-12T10:55:27,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:27,342 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111254f855cbc0d24dffa701d25ddf02b909_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111254f855cbc0d24dffa701d25ddf02b909_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/e2ed0dcb10a34d7b96bbc68a354d9f66, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/e2ed0dcb10a34d7b96bbc68a354d9f66 is 175, key is test_row_0/A:col10/1731408925944/Put/seqid=0 2024-11-12T10:55:27,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742226_1402 (size=31255) 2024-11-12T10:55:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T10:55:27,600 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#A#compaction#339 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:27,601 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/4d08597d0828414d87248f319986fdd1 is 175, key is test_row_0/A:col10/1731408924752/Put/seqid=0 2024-11-12T10:55:27,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742227_1403 (size=31685) 2024-11-12T10:55:27,770 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/e2ed0dcb10a34d7b96bbc68a354d9f66 2024-11-12T10:55:27,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/14462831cd9e451084312ffd8e77994c is 50, key is test_row_0/B:col10/1731408925944/Put/seqid=0 2024-11-12T10:55:27,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742228_1404 (size=12301) 2024-11-12T10:55:27,816 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/14462831cd9e451084312ffd8e77994c 2024-11-12T10:55:27,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/6eca2c497943471aa716c6f7ddbec494 is 50, key is test_row_0/C:col10/1731408925944/Put/seqid=0 2024-11-12T10:55:27,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742229_1405 (size=12301) 2024-11-12T10:55:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T10:55:28,014 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/4d08597d0828414d87248f319986fdd1 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/4d08597d0828414d87248f319986fdd1 2024-11-12T10:55:28,023 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/A of e8a1716ac64a4675f49f2e2b91b01012 into 4d08597d0828414d87248f319986fdd1(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:28,023 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:28,023 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/A, priority=13, startTime=1731408927165; duration=0sec 2024-11-12T10:55:28,024 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:28,024 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:A 2024-11-12T10:55:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:28,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:28,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408988152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408988153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408988154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,249 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/6eca2c497943471aa716c6f7ddbec494 2024-11-12T10:55:28,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/e2ed0dcb10a34d7b96bbc68a354d9f66 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/e2ed0dcb10a34d7b96bbc68a354d9f66 2024-11-12T10:55:28,255 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/e2ed0dcb10a34d7b96bbc68a354d9f66, entries=150, sequenceid=270, filesize=30.5 K 2024-11-12T10:55:28,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/14462831cd9e451084312ffd8e77994c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/14462831cd9e451084312ffd8e77994c 2024-11-12T10:55:28,258 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/14462831cd9e451084312ffd8e77994c, entries=150, sequenceid=270, filesize=12.0 K 2024-11-12T10:55:28,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/6eca2c497943471aa716c6f7ddbec494 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/6eca2c497943471aa716c6f7ddbec494 2024-11-12T10:55:28,262 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/6eca2c497943471aa716c6f7ddbec494, entries=150, sequenceid=270, filesize=12.0 K 2024-11-12T10:55:28,262 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e8a1716ac64a4675f49f2e2b91b01012 in 960ms, sequenceid=270, compaction requested=false 2024-11-12T10:55:28,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:28,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:28,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-12T10:55:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-12T10:55:28,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-12T10:55:28,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4220 sec 2024-11-12T10:55:28,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.4270 sec 2024-11-12T10:55:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:28,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T10:55:28,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:28,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:28,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:28,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:28,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:28,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:28,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112815f0effa94245228fbaeb466e2e9229_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408928153/Put/seqid=0 2024-11-12T10:55:28,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742230_1406 (size=14994) 2024-11-12T10:55:28,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408988305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408988306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408988312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408988417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408988427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408988427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408988636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408988636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408988647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,685 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:28,688 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112815f0effa94245228fbaeb466e2e9229_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112815f0effa94245228fbaeb466e2e9229_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:28,688 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/e3674aa8b63849e2807861ecc9b9ef7f, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:28,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/e3674aa8b63849e2807861ecc9b9ef7f is 175, key is test_row_0/A:col10/1731408928153/Put/seqid=0 2024-11-12T10:55:28,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742231_1407 (size=39949) 2024-11-12T10:55:28,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39328 deadline: 1731408988757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,760 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in e8a1716ac64a4675f49f2e2b91b01012 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18335 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1731408988807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,811 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in e8a1716ac64a4675f49f2e2b91b01012 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18394 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408988946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408988957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408988967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T10:55:28,987 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:28,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-12T10:55:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T10:55:28,990 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:28,990 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:28,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:29,092 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/e3674aa8b63849e2807861ecc9b9ef7f 2024-11-12T10:55:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T10:55:29,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/4ed7e0a9ff1b4fdea8ad2caac03476f5 is 50, key is test_row_0/B:col10/1731408928153/Put/seqid=0 2024-11-12T10:55:29,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742232_1408 (size=12301) 2024-11-12T10:55:29,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/4ed7e0a9ff1b4fdea8ad2caac03476f5 2024-11-12T10:55:29,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/ac7ce0f06ad14f8583030c29424e6748 is 50, key is test_row_0/C:col10/1731408928153/Put/seqid=0 2024-11-12T10:55:29,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742233_1409 (size=12301) 2024-11-12T10:55:29,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T10:55:29,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:29,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:29,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:29,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:29,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:29,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T10:55:29,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:29,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:29,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:29,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:29,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T10:55:29,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T10:55:29,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:29,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:29,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:29,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:29,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:29,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39250 deadline: 1731408989467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:29,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39284 deadline: 1731408989477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:29,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39260 deadline: 1731408989487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:29,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:29,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:29,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:55:29,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/ac7ce0f06ad14f8583030c29424e6748 2024-11-12T10:55:29,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/e3674aa8b63849e2807861ecc9b9ef7f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/e3674aa8b63849e2807861ecc9b9ef7f 2024-11-12T10:55:29,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/e3674aa8b63849e2807861ecc9b9ef7f, entries=200, sequenceid=286, filesize=39.0 K 2024-11-12T10:55:29,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/4ed7e0a9ff1b4fdea8ad2caac03476f5 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/4ed7e0a9ff1b4fdea8ad2caac03476f5 2024-11-12T10:55:29,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/4ed7e0a9ff1b4fdea8ad2caac03476f5, entries=150, sequenceid=286, filesize=12.0 K 2024-11-12T10:55:29,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/ac7ce0f06ad14f8583030c29424e6748 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/ac7ce0f06ad14f8583030c29424e6748 2024-11-12T10:55:29,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/ac7ce0f06ad14f8583030c29424e6748, entries=150, sequenceid=286, filesize=12.0 K 2024-11-12T10:55:29,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e8a1716ac64a4675f49f2e2b91b01012 in 1255ms, sequenceid=286, compaction requested=true 2024-11-12T10:55:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:55:29,529 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:29,529 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:55:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8a1716ac64a4675f49f2e2b91b01012:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:55:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:29,530 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:29,530 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/A is initiating minor compaction (all files) 2024-11-12T10:55:29,530 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/A in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:29,530 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/4d08597d0828414d87248f319986fdd1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/e2ed0dcb10a34d7b96bbc68a354d9f66, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/e3674aa8b63849e2807861ecc9b9ef7f] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=100.5 K 2024-11-12T10:55:29,530 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:29,530 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/4d08597d0828414d87248f319986fdd1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/e2ed0dcb10a34d7b96bbc68a354d9f66, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/e3674aa8b63849e2807861ecc9b9ef7f] 2024-11-12T10:55:29,530 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d08597d0828414d87248f319986fdd1, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731408924752 2024-11-12T10:55:29,530 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:29,530 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/B is initiating minor compaction (all files) 2024-11-12T10:55:29,530 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/B in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:29,531 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/5c40d80eb1a2498fb884fd3eb62921c3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/14462831cd9e451084312ffd8e77994c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/4ed7e0a9ff1b4fdea8ad2caac03476f5] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=36.5 K 2024-11-12T10:55:29,531 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c40d80eb1a2498fb884fd3eb62921c3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731408924752 2024-11-12T10:55:29,531 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2ed0dcb10a34d7b96bbc68a354d9f66, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731408925944 2024-11-12T10:55:29,531 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 14462831cd9e451084312ffd8e77994c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731408925944 2024-11-12T10:55:29,531 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3674aa8b63849e2807861ecc9b9ef7f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731408928142 2024-11-12T10:55:29,531 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ed7e0a9ff1b4fdea8ad2caac03476f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731408928142 2024-11-12T10:55:29,536 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:29,541 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112ec03280c7bd04fe997039a26482166b7_e8a1716ac64a4675f49f2e2b91b01012 store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:29,543 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112ec03280c7bd04fe997039a26482166b7_e8a1716ac64a4675f49f2e2b91b01012, store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:29,543 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ec03280c7bd04fe997039a26482166b7_e8a1716ac64a4675f49f2e2b91b01012 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:29,543 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#B#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:29,544 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/fc8a6d71f1094ced9614a72d213558bb is 50, key is test_row_0/B:col10/1731408928153/Put/seqid=0 2024-11-12T10:55:29,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742235_1411 (size=12983) 2024-11-12T10:55:29,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742234_1410 (size=4469) 2024-11-12T10:55:29,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T10:55:29,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:29,600 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T10:55:29,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:29,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:29,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:29,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:29,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:29,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:29,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123f9c4f894e274b1cae55b0694ce90db2_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408928292/Put/seqid=0 2024-11-12T10:55:29,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742236_1412 (size=12454) 2024-11-12T10:55:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T10:55:29,958 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/fc8a6d71f1094ced9614a72d213558bb as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/fc8a6d71f1094ced9614a72d213558bb 2024-11-12T10:55:29,961 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#A#compaction#347 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:29,962 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/aacd481078054a679d7fb7038e61502b is 175, key is test_row_0/A:col10/1731408928153/Put/seqid=0 2024-11-12T10:55:29,963 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/B of e8a1716ac64a4675f49f2e2b91b01012 into fc8a6d71f1094ced9614a72d213558bb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:29,963 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:29,963 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/B, priority=13, startTime=1731408929529; duration=0sec 2024-11-12T10:55:29,963 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:29,963 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:B 2024-11-12T10:55:29,963 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:29,964 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:29,964 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): e8a1716ac64a4675f49f2e2b91b01012/C is initiating minor compaction (all files) 2024-11-12T10:55:29,964 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8a1716ac64a4675f49f2e2b91b01012/C in TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:29,964 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/509b0c24ec754b0f924809d00154ebd7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/6eca2c497943471aa716c6f7ddbec494, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/ac7ce0f06ad14f8583030c29424e6748] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp, totalSize=36.5 K 2024-11-12T10:55:29,964 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 509b0c24ec754b0f924809d00154ebd7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731408924752 2024-11-12T10:55:29,965 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eca2c497943471aa716c6f7ddbec494, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731408925944 2024-11-12T10:55:29,965 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ac7ce0f06ad14f8583030c29424e6748, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731408928142 2024-11-12T10:55:29,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742237_1413 (size=31937) 2024-11-12T10:55:29,970 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/aacd481078054a679d7fb7038e61502b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/aacd481078054a679d7fb7038e61502b 2024-11-12T10:55:29,973 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8a1716ac64a4675f49f2e2b91b01012#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:29,974 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/b040bd5aca6142d8ba007f293947d418 is 50, key is test_row_0/C:col10/1731408928153/Put/seqid=0 2024-11-12T10:55:29,975 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/A of e8a1716ac64a4675f49f2e2b91b01012 into aacd481078054a679d7fb7038e61502b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:29,975 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:29,975 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/A, priority=13, startTime=1731408929529; duration=0sec 2024-11-12T10:55:29,975 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:29,975 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:A 2024-11-12T10:55:29,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742238_1414 (size=12983) 2024-11-12T10:55:30,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:30,014 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123f9c4f894e274b1cae55b0694ce90db2_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123f9c4f894e274b1cae55b0694ce90db2_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:30,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9b0d2ddafe904b9396d24af1755017e3, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:30,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9b0d2ddafe904b9396d24af1755017e3 is 175, key is test_row_0/A:col10/1731408928292/Put/seqid=0 2024-11-12T10:55:30,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742239_1415 (size=31255) 2024-11-12T10:55:30,023 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9b0d2ddafe904b9396d24af1755017e3 2024-11-12T10:55:30,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/42f7945310504658815654fc4d68cf95 is 50, key is test_row_0/B:col10/1731408928292/Put/seqid=0 2024-11-12T10:55:30,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742240_1416 (size=12301) 2024-11-12T10:55:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T10:55:30,381 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/b040bd5aca6142d8ba007f293947d418 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/b040bd5aca6142d8ba007f293947d418 2024-11-12T10:55:30,384 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8a1716ac64a4675f49f2e2b91b01012/C of e8a1716ac64a4675f49f2e2b91b01012 into b040bd5aca6142d8ba007f293947d418(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:30,384 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:30,384 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., storeName=e8a1716ac64a4675f49f2e2b91b01012/C, priority=13, startTime=1731408929529; duration=0sec 2024-11-12T10:55:30,385 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:30,385 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8a1716ac64a4675f49f2e2b91b01012:C 2024-11-12T10:55:30,427 INFO [Thread-1523 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1523. 2024-11-12T10:55:30,427 DEBUG [Thread-1523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:55:30,427 DEBUG [Thread-1523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,427 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:55:30,427 DEBUG [Thread-1523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,429 INFO [Thread-1527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1527. 2024-11-12T10:55:30,429 DEBUG [Thread-1527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:55:30,429 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,429 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,429 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:55:30,430 INFO [Thread-1525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1525. 2024-11-12T10:55:30,430 DEBUG [Thread-1525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:55:30,430 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,430 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:55:30,433 INFO [Thread-1521 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1521. 2024-11-12T10:55:30,433 DEBUG [Thread-1521 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:55:30,433 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,433 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,433 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:55:30,433 INFO [Thread-1519 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1519. 2024-11-12T10:55:30,433 DEBUG [Thread-1519 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:55:30,433 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,434 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,434 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:55:30,438 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/42f7945310504658815654fc4d68cf95 2024-11-12T10:55:30,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/7178ac2a99d949719b15ba488c877f81 is 50, key is test_row_0/C:col10/1731408928292/Put/seqid=0 2024-11-12T10:55:30,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742241_1417 (size=12301) 2024-11-12T10:55:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:30,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. as already flushing 2024-11-12T10:55:30,489 INFO [Thread-1510 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1510. 2024-11-12T10:55:30,489 DEBUG [Thread-1510 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:55:30,489 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,490 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,490 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:55:30,500 INFO [Thread-1514 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1514. 2024-11-12T10:55:30,501 DEBUG [Thread-1514 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:55:30,501 DEBUG [Thread-1514 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,501 INFO [Thread-1516 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1516. 2024-11-12T10:55:30,501 DEBUG [Thread-1514 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:55:30,501 DEBUG [Thread-1516 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:55:30,501 DEBUG [Thread-1516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,501 DEBUG [Thread-1516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:30,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:55:30,848 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/7178ac2a99d949719b15ba488c877f81 2024-11-12T10:55:30,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9b0d2ddafe904b9396d24af1755017e3 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9b0d2ddafe904b9396d24af1755017e3 2024-11-12T10:55:30,858 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9b0d2ddafe904b9396d24af1755017e3, entries=150, sequenceid=303, filesize=30.5 K 2024-11-12T10:55:30,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/42f7945310504658815654fc4d68cf95 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/42f7945310504658815654fc4d68cf95 2024-11-12T10:55:30,862 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/42f7945310504658815654fc4d68cf95, entries=150, sequenceid=303, filesize=12.0 K 2024-11-12T10:55:30,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/7178ac2a99d949719b15ba488c877f81 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/7178ac2a99d949719b15ba488c877f81 2024-11-12T10:55:30,866 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/7178ac2a99d949719b15ba488c877f81, entries=150, sequenceid=303, filesize=12.0 K 2024-11-12T10:55:30,867 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=20.13 KB/20610 for e8a1716ac64a4675f49f2e2b91b01012 in 1267ms, sequenceid=303, compaction requested=false 2024-11-12T10:55:30,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for e8a1716ac64a4675f49f2e2b91b01012: 2024-11-12T10:55:30,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:30,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-12T10:55:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-12T10:55:30,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-12T10:55:30,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8780 sec 2024-11-12T10:55:30,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8820 sec 2024-11-12T10:55:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T10:55:31,148 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:31,378 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T10:55:35,516 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9e579ce4237844bfa3352d39d03bc90d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/77bf59e672e24f39a1855feb1a8d76ab, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/99cbd98532d14629abf05ded8f258d94, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/d21d87b96da9424ea09626093b546df1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/f6eb9b7c8ff84b9e855ee8a76a353439, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/1d35620272d0405896822f8bb487c797, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/8f0e732799da40ea89e8af949fdc9448, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/7a7582e9b16a4ad7a1912221923a0943, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9f19c89678514ec9aa4570618e166438, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/32f3f9ea84204911a9b8d805d9e1be6e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/ed7cf3acae8f44acb7c059305ad3f96a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/21bbc515c56e46118e76b9d0d4065efa, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/bfa8499c8f39408b871853233a771f96, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/4060739899364b9d95cdeda35fac6690, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/7b2ec1f0b3fa440e86a020d7ffe2d12e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/f053eb344e94439594b468c409482150, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/8d848cf5e6aa4d0782e6a3a5b5bfda29, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/4d08597d0828414d87248f319986fdd1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/e2ed0dcb10a34d7b96bbc68a354d9f66, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/e3674aa8b63849e2807861ecc9b9ef7f] to archive 2024-11-12T10:55:35,518 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:55:35,519 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9e579ce4237844bfa3352d39d03bc90d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9e579ce4237844bfa3352d39d03bc90d 2024-11-12T10:55:35,521 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/77bf59e672e24f39a1855feb1a8d76ab to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/77bf59e672e24f39a1855feb1a8d76ab 2024-11-12T10:55:35,522 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/99cbd98532d14629abf05ded8f258d94 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/99cbd98532d14629abf05ded8f258d94 2024-11-12T10:55:35,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/d21d87b96da9424ea09626093b546df1 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/d21d87b96da9424ea09626093b546df1 2024-11-12T10:55:35,524 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/f6eb9b7c8ff84b9e855ee8a76a353439 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/f6eb9b7c8ff84b9e855ee8a76a353439 2024-11-12T10:55:35,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/1d35620272d0405896822f8bb487c797 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/1d35620272d0405896822f8bb487c797 2024-11-12T10:55:35,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/8f0e732799da40ea89e8af949fdc9448 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/8f0e732799da40ea89e8af949fdc9448 2024-11-12T10:55:35,526 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/7a7582e9b16a4ad7a1912221923a0943 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/7a7582e9b16a4ad7a1912221923a0943 2024-11-12T10:55:35,527 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9f19c89678514ec9aa4570618e166438 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9f19c89678514ec9aa4570618e166438 2024-11-12T10:55:35,528 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/32f3f9ea84204911a9b8d805d9e1be6e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/32f3f9ea84204911a9b8d805d9e1be6e 2024-11-12T10:55:35,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/ed7cf3acae8f44acb7c059305ad3f96a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/ed7cf3acae8f44acb7c059305ad3f96a 2024-11-12T10:55:35,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/21bbc515c56e46118e76b9d0d4065efa to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/21bbc515c56e46118e76b9d0d4065efa 2024-11-12T10:55:35,530 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/bfa8499c8f39408b871853233a771f96 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/bfa8499c8f39408b871853233a771f96 2024-11-12T10:55:35,531 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/4060739899364b9d95cdeda35fac6690 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/4060739899364b9d95cdeda35fac6690 2024-11-12T10:55:35,532 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/7b2ec1f0b3fa440e86a020d7ffe2d12e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/7b2ec1f0b3fa440e86a020d7ffe2d12e 2024-11-12T10:55:35,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/f053eb344e94439594b468c409482150 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/f053eb344e94439594b468c409482150 2024-11-12T10:55:35,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/8d848cf5e6aa4d0782e6a3a5b5bfda29 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/8d848cf5e6aa4d0782e6a3a5b5bfda29 2024-11-12T10:55:35,534 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/4d08597d0828414d87248f319986fdd1 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/4d08597d0828414d87248f319986fdd1 2024-11-12T10:55:35,535 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/e2ed0dcb10a34d7b96bbc68a354d9f66 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/e2ed0dcb10a34d7b96bbc68a354d9f66 2024-11-12T10:55:35,536 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/e3674aa8b63849e2807861ecc9b9ef7f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/e3674aa8b63849e2807861ecc9b9ef7f 2024-11-12T10:55:35,538 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/b92558606a824ca9bcfa121b2c33ce54, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/152c82beca824887a13e66995905995f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/3943bd01d60e4595b4625fcb3765218b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/4a64782477fd4c28b734338953e5af29, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/df07bc0674ee4d6e8dc6e8ed873cfe50, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/18af48d68b114756b934de995ca7a89f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/e0f29a2b523b4056808690ba861bc677, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/17931ffc99be4c39bfda5d441718bedf, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/a2f81a49d847491ca5aba15c03e225ec, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/dd6648126c5e42a884353077afc085b1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/aee152b102f940daa0c2923b53577725, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/558c0d47cc1a4671bdfce45a6abe1b72, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/5cafe6a546764f86a0687c55409ff9b5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/e3dbd21526724934ba0460ff60d74cb0, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/daa69a24ac894d8090158710ac652085, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/003fcf0c7a1840d288c5992aba7e6ab6, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/5c40d80eb1a2498fb884fd3eb62921c3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/0d61b817a07d4951b121e593922878b1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/14462831cd9e451084312ffd8e77994c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/4ed7e0a9ff1b4fdea8ad2caac03476f5] to archive 2024-11-12T10:55:35,539 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:55:35,540 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/b92558606a824ca9bcfa121b2c33ce54 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/b92558606a824ca9bcfa121b2c33ce54 2024-11-12T10:55:35,541 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/152c82beca824887a13e66995905995f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/152c82beca824887a13e66995905995f 2024-11-12T10:55:35,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/3943bd01d60e4595b4625fcb3765218b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/3943bd01d60e4595b4625fcb3765218b 2024-11-12T10:55:35,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/4a64782477fd4c28b734338953e5af29 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/4a64782477fd4c28b734338953e5af29 2024-11-12T10:55:35,543 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/df07bc0674ee4d6e8dc6e8ed873cfe50 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/df07bc0674ee4d6e8dc6e8ed873cfe50 2024-11-12T10:55:35,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/18af48d68b114756b934de995ca7a89f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/18af48d68b114756b934de995ca7a89f 2024-11-12T10:55:35,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/e0f29a2b523b4056808690ba861bc677 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/e0f29a2b523b4056808690ba861bc677 2024-11-12T10:55:35,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/17931ffc99be4c39bfda5d441718bedf to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/17931ffc99be4c39bfda5d441718bedf 2024-11-12T10:55:35,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/a2f81a49d847491ca5aba15c03e225ec to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/a2f81a49d847491ca5aba15c03e225ec 2024-11-12T10:55:35,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/dd6648126c5e42a884353077afc085b1 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/dd6648126c5e42a884353077afc085b1 2024-11-12T10:55:35,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/aee152b102f940daa0c2923b53577725 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/aee152b102f940daa0c2923b53577725 2024-11-12T10:55:35,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/558c0d47cc1a4671bdfce45a6abe1b72 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/558c0d47cc1a4671bdfce45a6abe1b72 2024-11-12T10:55:35,548 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/5cafe6a546764f86a0687c55409ff9b5 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/5cafe6a546764f86a0687c55409ff9b5 2024-11-12T10:55:35,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/e3dbd21526724934ba0460ff60d74cb0 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/e3dbd21526724934ba0460ff60d74cb0 2024-11-12T10:55:35,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/daa69a24ac894d8090158710ac652085 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/daa69a24ac894d8090158710ac652085 2024-11-12T10:55:35,550 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/003fcf0c7a1840d288c5992aba7e6ab6 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/003fcf0c7a1840d288c5992aba7e6ab6 2024-11-12T10:55:35,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/5c40d80eb1a2498fb884fd3eb62921c3 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/5c40d80eb1a2498fb884fd3eb62921c3 2024-11-12T10:55:35,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/0d61b817a07d4951b121e593922878b1 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/0d61b817a07d4951b121e593922878b1 2024-11-12T10:55:35,553 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/14462831cd9e451084312ffd8e77994c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/14462831cd9e451084312ffd8e77994c 2024-11-12T10:55:35,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/4ed7e0a9ff1b4fdea8ad2caac03476f5 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/4ed7e0a9ff1b4fdea8ad2caac03476f5 2024-11-12T10:55:35,558 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/2b10926c0cee4d449d6218ff7a85afff, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/93ccc68dfe7648e69312cd85c7c0ac9d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/e9ca571ac84442f39f5ada8104a5fea4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/9ac9ba1495654531a97b09780e2a9481, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3bdbb4cb4b664da4a777129f024fca2b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3133f53f1a914904b61de251ea14aa8a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/fdaa044f58944e5fbd17f2b6f8c28582, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/7d11a64be1ad45f4bbe4e74a32e9ce3d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/957c3aeb6c2643d88cfced990e092e2c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/530a31ad6c664bc996cea4d738ffed30, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/20f87b8c24214837ace71f64d7d0b795, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/38b952c6cba04722a6d2842d9764b64e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/ef337fdcb8874fd5ac8690b2391aec3f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3cad5edd794243c6b6b3534575570607, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/22e99088a9914c18a4b12c4e9ad92065, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/80da0ea8988041dd9755bebf750a92c4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/509b0c24ec754b0f924809d00154ebd7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/9819e133aec94be4b9444b00abbf6384, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/6eca2c497943471aa716c6f7ddbec494, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/ac7ce0f06ad14f8583030c29424e6748] to archive 2024-11-12T10:55:35,558 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:55:35,560 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/2b10926c0cee4d449d6218ff7a85afff to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/2b10926c0cee4d449d6218ff7a85afff 2024-11-12T10:55:35,561 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/93ccc68dfe7648e69312cd85c7c0ac9d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/93ccc68dfe7648e69312cd85c7c0ac9d 2024-11-12T10:55:35,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/e9ca571ac84442f39f5ada8104a5fea4 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/e9ca571ac84442f39f5ada8104a5fea4 2024-11-12T10:55:35,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/9ac9ba1495654531a97b09780e2a9481 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/9ac9ba1495654531a97b09780e2a9481 2024-11-12T10:55:35,563 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3bdbb4cb4b664da4a777129f024fca2b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3bdbb4cb4b664da4a777129f024fca2b 2024-11-12T10:55:35,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3133f53f1a914904b61de251ea14aa8a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3133f53f1a914904b61de251ea14aa8a 2024-11-12T10:55:35,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/fdaa044f58944e5fbd17f2b6f8c28582 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/fdaa044f58944e5fbd17f2b6f8c28582 2024-11-12T10:55:35,565 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/7d11a64be1ad45f4bbe4e74a32e9ce3d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/7d11a64be1ad45f4bbe4e74a32e9ce3d 2024-11-12T10:55:35,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/957c3aeb6c2643d88cfced990e092e2c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/957c3aeb6c2643d88cfced990e092e2c 2024-11-12T10:55:35,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/530a31ad6c664bc996cea4d738ffed30 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/530a31ad6c664bc996cea4d738ffed30 2024-11-12T10:55:35,567 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/20f87b8c24214837ace71f64d7d0b795 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/20f87b8c24214837ace71f64d7d0b795 2024-11-12T10:55:35,568 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/38b952c6cba04722a6d2842d9764b64e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/38b952c6cba04722a6d2842d9764b64e 2024-11-12T10:55:35,568 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/ef337fdcb8874fd5ac8690b2391aec3f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/ef337fdcb8874fd5ac8690b2391aec3f 2024-11-12T10:55:35,569 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3cad5edd794243c6b6b3534575570607 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/3cad5edd794243c6b6b3534575570607 2024-11-12T10:55:35,569 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/22e99088a9914c18a4b12c4e9ad92065 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/22e99088a9914c18a4b12c4e9ad92065 2024-11-12T10:55:35,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/80da0ea8988041dd9755bebf750a92c4 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/80da0ea8988041dd9755bebf750a92c4 2024-11-12T10:55:35,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/509b0c24ec754b0f924809d00154ebd7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/509b0c24ec754b0f924809d00154ebd7 2024-11-12T10:55:35,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/9819e133aec94be4b9444b00abbf6384 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/9819e133aec94be4b9444b00abbf6384 2024-11-12T10:55:35,572 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/6eca2c497943471aa716c6f7ddbec494 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/6eca2c497943471aa716c6f7ddbec494 2024-11-12T10:55:35,573 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/aed202db7571:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/ac7ce0f06ad14f8583030c29424e6748 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/ac7ce0f06ad14f8583030c29424e6748 2024-11-12T10:55:38,829 INFO [Thread-1508 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1508. 2024-11-12T10:55:38,829 DEBUG [Thread-1508 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:55:38,829 DEBUG [Thread-1508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:38,830 DEBUG [Thread-1508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:38,830 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:55:38,859 INFO [Thread-1512 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1512. 2024-11-12T10:55:38,859 DEBUG [Thread-1512 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:55:38,859 DEBUG [Thread-1512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:38,859 DEBUG [Thread-1512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:38,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 1 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 1 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1069 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3207 rows 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1072 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3216 rows 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1075 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3225 rows 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1072 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3216 rows 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1065 2024-11-12T10:55:38,860 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3195 rows 2024-11-12T10:55:38,860 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T10:55:38,860 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T10:55:38,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:38,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:38,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:55:38,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T10:55:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T10:55:38,868 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408938868"}]},"ts":"1731408938868"} 2024-11-12T10:55:38,870 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T10:55:38,870 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T10:55:38,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T10:55:38,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, UNASSIGN}] 2024-11-12T10:55:38,878 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, UNASSIGN 2024-11-12T10:55:38,879 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=e8a1716ac64a4675f49f2e2b91b01012, regionState=CLOSING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:55:38,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, UNASSIGN because future has completed 2024-11-12T10:55:38,881 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T10:55:38,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:55:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T10:55:39,034 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:39,034 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T10:55:39,034 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing e8a1716ac64a4675f49f2e2b91b01012, disabling compactions & flushes 2024-11-12T10:55:39,034 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:39,034 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:39,034 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. after waiting 0 ms 2024-11-12T10:55:39,034 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:39,034 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing e8a1716ac64a4675f49f2e2b91b01012 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T10:55:39,035 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=A 2024-11-12T10:55:39,035 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:39,035 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=B 2024-11-12T10:55:39,035 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:39,035 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8a1716ac64a4675f49f2e2b91b01012, store=C 2024-11-12T10:55:39,035 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:39,040 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f0e59ba318394f7e9c2f868ce7e310b2_e8a1716ac64a4675f49f2e2b91b01012 is 50, key is test_row_0/A:col10/1731408938857/Put/seqid=0 2024-11-12T10:55:39,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742242_1418 (size=12454) 2024-11-12T10:55:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T10:55:39,444 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:39,450 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f0e59ba318394f7e9c2f868ce7e310b2_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f0e59ba318394f7e9c2f868ce7e310b2_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:39,451 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9149125a0b8544f1b0b9d46e621a0f08, store: [table=TestAcidGuarantees family=A region=e8a1716ac64a4675f49f2e2b91b01012] 2024-11-12T10:55:39,451 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9149125a0b8544f1b0b9d46e621a0f08 is 175, key is test_row_0/A:col10/1731408938857/Put/seqid=0 2024-11-12T10:55:39,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742243_1419 (size=31255) 2024-11-12T10:55:39,455 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9149125a0b8544f1b0b9d46e621a0f08 2024-11-12T10:55:39,461 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/c93d468d90624271bd67cbafe554ff69 is 50, key is test_row_0/B:col10/1731408938857/Put/seqid=0 2024-11-12T10:55:39,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742244_1420 (size=12301) 2024-11-12T10:55:39,465 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/c93d468d90624271bd67cbafe554ff69 2024-11-12T10:55:39,471 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/93cb4c7a8b2c462eb96a50daf924451e is 50, key is test_row_0/C:col10/1731408938857/Put/seqid=0 2024-11-12T10:55:39,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742245_1421 (size=12301) 2024-11-12T10:55:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T10:55:39,878 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/93cb4c7a8b2c462eb96a50daf924451e 2024-11-12T10:55:39,883 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/A/9149125a0b8544f1b0b9d46e621a0f08 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9149125a0b8544f1b0b9d46e621a0f08 2024-11-12T10:55:39,886 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9149125a0b8544f1b0b9d46e621a0f08, entries=150, sequenceid=314, filesize=30.5 K 2024-11-12T10:55:39,887 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/B/c93d468d90624271bd67cbafe554ff69 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/c93d468d90624271bd67cbafe554ff69 2024-11-12T10:55:39,890 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/c93d468d90624271bd67cbafe554ff69, entries=150, sequenceid=314, filesize=12.0 K 2024-11-12T10:55:39,890 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/.tmp/C/93cb4c7a8b2c462eb96a50daf924451e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/93cb4c7a8b2c462eb96a50daf924451e 2024-11-12T10:55:39,893 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/93cb4c7a8b2c462eb96a50daf924451e, entries=150, sequenceid=314, filesize=12.0 K 2024-11-12T10:55:39,894 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e8a1716ac64a4675f49f2e2b91b01012 in 860ms, sequenceid=314, compaction requested=true 2024-11-12T10:55:39,901 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/recovered.edits/317.seqid, newMaxSeqId=317, maxSeqId=4 2024-11-12T10:55:39,902 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. 2024-11-12T10:55:39,902 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for e8a1716ac64a4675f49f2e2b91b01012: Waiting for close lock at 1731408939034Running coprocessor pre-close hooks at 1731408939034Disabling compacts and flushes for region at 1731408939034Disabling writes for close at 1731408939034Obtaining lock to block concurrent updates at 1731408939034Preparing flush snapshotting stores in e8a1716ac64a4675f49f2e2b91b01012 at 1731408939034Finished memstore snapshotting TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731408939035 (+1 ms)Flushing stores of TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012. at 1731408939035Flushing e8a1716ac64a4675f49f2e2b91b01012/A: creating writer at 1731408939035Flushing mob file e8a1716ac64a4675f49f2e2b91b01012/A: appending metadata at 1731408939040 (+5 ms)Flushing mob file e8a1716ac64a4675f49f2e2b91b01012/A: closing flushed file at 1731408939040Flushing e8a1716ac64a4675f49f2e2b91b01012/A: appending metadata at 1731408939451 (+411 ms)Flushing e8a1716ac64a4675f49f2e2b91b01012/A: closing flushed file at 1731408939451Flushing e8a1716ac64a4675f49f2e2b91b01012/B: creating writer at 1731408939458 (+7 ms)Flushing e8a1716ac64a4675f49f2e2b91b01012/B: appending metadata at 1731408939461 (+3 ms)Flushing e8a1716ac64a4675f49f2e2b91b01012/B: closing flushed file at 1731408939461Flushing e8a1716ac64a4675f49f2e2b91b01012/C: creating writer at 1731408939468 (+7 ms)Flushing e8a1716ac64a4675f49f2e2b91b01012/C: appending metadata at 1731408939470 (+2 ms)Flushing e8a1716ac64a4675f49f2e2b91b01012/C: closing flushed file at 1731408939470Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c53b035: reopening flushed file at 1731408939882 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4994b933: reopening flushed file at 1731408939886 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54f5f43e: reopening flushed file at 1731408939890 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e8a1716ac64a4675f49f2e2b91b01012 in 860ms, sequenceid=314, compaction requested=true at 1731408939894 (+4 ms)Writing region close event to WAL at 1731408939895 (+1 ms)Running coprocessor post-close hooks at 1731408939902 (+7 ms)Closed at 1731408939902 2024-11-12T10:55:39,904 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:39,905 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=e8a1716ac64a4675f49f2e2b91b01012, regionState=CLOSED 2024-11-12T10:55:39,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:55:39,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-12T10:55:39,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure e8a1716ac64a4675f49f2e2b91b01012, server=aed202db7571,43143,1731408812834 in 1.0260 sec 2024-11-12T10:55:39,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-12T10:55:39,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8a1716ac64a4675f49f2e2b91b01012, UNASSIGN in 1.0350 sec 2024-11-12T10:55:39,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-12T10:55:39,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0400 sec 2024-11-12T10:55:39,914 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408939913"}]},"ts":"1731408939913"} 2024-11-12T10:55:39,915 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T10:55:39,915 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T10:55:39,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.0550 sec 2024-11-12T10:55:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T10:55:40,008 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:40,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T10:55:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:40,010 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T10:55:40,011 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:40,013 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,016 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/recovered.edits] 2024-11-12T10:55:40,018 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9149125a0b8544f1b0b9d46e621a0f08 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9149125a0b8544f1b0b9d46e621a0f08 2024-11-12T10:55:40,019 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9b0d2ddafe904b9396d24af1755017e3 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/9b0d2ddafe904b9396d24af1755017e3 2024-11-12T10:55:40,020 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/aacd481078054a679d7fb7038e61502b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/A/aacd481078054a679d7fb7038e61502b 2024-11-12T10:55:40,022 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/42f7945310504658815654fc4d68cf95 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/42f7945310504658815654fc4d68cf95 2024-11-12T10:55:40,023 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/c93d468d90624271bd67cbafe554ff69 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/c93d468d90624271bd67cbafe554ff69 2024-11-12T10:55:40,024 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/fc8a6d71f1094ced9614a72d213558bb to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/B/fc8a6d71f1094ced9614a72d213558bb 2024-11-12T10:55:40,026 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/7178ac2a99d949719b15ba488c877f81 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/7178ac2a99d949719b15ba488c877f81 2024-11-12T10:55:40,027 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/93cb4c7a8b2c462eb96a50daf924451e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/93cb4c7a8b2c462eb96a50daf924451e 2024-11-12T10:55:40,028 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/b040bd5aca6142d8ba007f293947d418 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/C/b040bd5aca6142d8ba007f293947d418 2024-11-12T10:55:40,030 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/recovered.edits/317.seqid to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012/recovered.edits/317.seqid 2024-11-12T10:55:40,031 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,031 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T10:55:40,031 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T10:55:40,032 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T10:55:40,034 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111203e6c877839d46c99047d5a37ea042c1_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111203e6c877839d46c99047d5a37ea042c1_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,035 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122b7796aa1e4d4c9fa357531823e68044_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122b7796aa1e4d4c9fa357531823e68044_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,036 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112338b2d5da2e642d19b7935072fc7cb51_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112338b2d5da2e642d19b7935072fc7cb51_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,037 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123f9c4f894e274b1cae55b0694ce90db2_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123f9c4f894e274b1cae55b0694ce90db2_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,038 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112427820c37f6f4971b434bba37ae5dfd3_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112427820c37f6f4971b434bba37ae5dfd3_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,039 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111247569bfe0c814db8bb0120f4eeb82877_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111247569bfe0c814db8bb0120f4eeb82877_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,040 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111253129ff1a3b34031a2a747665b0fa20f_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111253129ff1a3b34031a2a747665b0fa20f_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,041 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111254f855cbc0d24dffa701d25ddf02b909_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111254f855cbc0d24dffa701d25ddf02b909_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,042 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125cbf843ef055441f85c35b6dddf9f867_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125cbf843ef055441f85c35b6dddf9f867_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,043 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127095e805e99849bb8c0d5a2fdef419f0_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127095e805e99849bb8c0d5a2fdef419f0_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,044 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112815f0effa94245228fbaeb466e2e9229_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112815f0effa94245228fbaeb466e2e9229_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,045 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128b819e60ec854c60b12a6946d702fb3f_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128b819e60ec854c60b12a6946d702fb3f_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,046 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ba379b88d9ce495e9bc96b06b1e85cd6_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ba379b88d9ce495e9bc96b06b1e85cd6_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,047 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d37ca085cc1c486ca31489b7dc7f9019_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d37ca085cc1c486ca31489b7dc7f9019_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,048 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d64d80fa95834a018eb90403811c78d6_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d64d80fa95834a018eb90403811c78d6_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,049 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d9837928f8fc404ca5e7e8d963a82c5f_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d9837928f8fc404ca5e7e8d963a82c5f_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,050 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f0e59ba318394f7e9c2f868ce7e310b2_e8a1716ac64a4675f49f2e2b91b01012 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f0e59ba318394f7e9c2f868ce7e310b2_e8a1716ac64a4675f49f2e2b91b01012 2024-11-12T10:55:40,050 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T10:55:40,053 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:40,055 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T10:55:40,057 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T10:55:40,059 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:40,059 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T10:55:40,059 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731408940059"}]},"ts":"9223372036854775807"} 2024-11-12T10:55:40,061 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T10:55:40,061 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e8a1716ac64a4675f49f2e2b91b01012, NAME => 'TestAcidGuarantees,,1731408908195.e8a1716ac64a4675f49f2e2b91b01012.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T10:55:40,061 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T10:55:40,061 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731408940061"}]},"ts":"9223372036854775807"} 2024-11-12T10:55:40,063 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T10:55:40,063 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:40,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-11-12T10:55:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T10:55:40,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T10:55:40,122 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:40,129 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 246) - Thread LEAK? -, OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=488 (was 573), ProcessCount=11 (was 11), AvailableMemoryMB=3076 (was 1848) - AvailableMemoryMB LEAK? - 2024-11-12T10:55:40,138 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=488, ProcessCount=11, AvailableMemoryMB=3075 2024-11-12T10:55:40,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T10:55:40,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T10:55:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T10:55:40,142 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T10:55:40,142 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:40,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-12T10:55:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-12T10:55:40,143 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T10:55:40,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742246_1422 (size=1061) 2024-11-12T10:55:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-12T10:55:40,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-12T10:55:40,550 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98 2024-11-12T10:55:40,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742247_1423 (size=53) 2024-11-12T10:55:40,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-12T10:55:40,955 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:55:40,955 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5b5c37b8175d8b0adea03f092f215997, disabling compactions & flushes 2024-11-12T10:55:40,955 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:40,955 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:40,955 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. after waiting 0 ms 2024-11-12T10:55:40,955 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:40,955 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:40,955 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5b5c37b8175d8b0adea03f092f215997: Waiting for close lock at 1731408940955Disabling compacts and flushes for region at 1731408940955Disabling writes for close at 1731408940955Writing region close event to WAL at 1731408940955Closed at 1731408940955 2024-11-12T10:55:40,956 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T10:55:40,956 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731408940956"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731408940956"}]},"ts":"1731408940956"} 2024-11-12T10:55:40,958 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T10:55:40,958 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T10:55:40,959 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408940958"}]},"ts":"1731408940958"} 2024-11-12T10:55:40,960 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T10:55:40,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b5c37b8175d8b0adea03f092f215997, ASSIGN}] 2024-11-12T10:55:40,961 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b5c37b8175d8b0adea03f092f215997, ASSIGN 2024-11-12T10:55:40,962 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b5c37b8175d8b0adea03f092f215997, ASSIGN; state=OFFLINE, location=aed202db7571,43143,1731408812834; forceNewPlan=false, retain=false 2024-11-12T10:55:41,113 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=5b5c37b8175d8b0adea03f092f215997, regionState=OPENING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:55:41,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b5c37b8175d8b0adea03f092f215997, ASSIGN because future has completed 2024-11-12T10:55:41,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:55:41,270 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:41,270 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:55:41,271 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:41,271 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:55:41,271 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:41,271 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-12T10:55:41,284 INFO [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:41,286 INFO [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:55:41,286 INFO [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b5c37b8175d8b0adea03f092f215997 columnFamilyName A 2024-11-12T10:55:41,286 DEBUG [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:41,287 INFO [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] regionserver.HStore(327): Store=5b5c37b8175d8b0adea03f092f215997/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:55:41,287 INFO [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:41,288 INFO [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:55:41,288 INFO [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b5c37b8175d8b0adea03f092f215997 columnFamilyName B 2024-11-12T10:55:41,288 DEBUG [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:41,290 INFO [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] regionserver.HStore(327): Store=5b5c37b8175d8b0adea03f092f215997/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:55:41,290 INFO [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:41,291 INFO [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:55:41,292 INFO [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b5c37b8175d8b0adea03f092f215997 columnFamilyName C 2024-11-12T10:55:41,292 DEBUG [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:55:41,292 INFO [StoreOpener-5b5c37b8175d8b0adea03f092f215997-1 {}] regionserver.HStore(327): Store=5b5c37b8175d8b0adea03f092f215997/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:55:41,292 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:41,293 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:41,293 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:41,294 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:41,295 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:41,295 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:41,295 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T10:55:41,296 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:41,301 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:55:41,301 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 5b5c37b8175d8b0adea03f092f215997; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66051720, jitterRate=-0.015752673149108887}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T10:55:41,301 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:41,302 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 5b5c37b8175d8b0adea03f092f215997: Running coprocessor pre-open hook at 1731408941271Writing region info on filesystem at 1731408941271Initializing all the Stores at 1731408941278 (+7 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408941278Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408941284 (+6 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408941284Cleaning up temporary data from old regions at 1731408941295 (+11 ms)Running coprocessor post-open hooks at 1731408941301 (+6 ms)Region opened successfully at 1731408941302 (+1 ms) 2024-11-12T10:55:41,304 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., pid=120, masterSystemTime=1731408941266 2024-11-12T10:55:41,307 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=5b5c37b8175d8b0adea03f092f215997, regionState=OPEN, openSeqNum=2, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:55:41,309 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:41,309 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:41,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:55:41,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-12T10:55:41,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 in 196 msec 2024-11-12T10:55:41,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-12T10:55:41,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b5c37b8175d8b0adea03f092f215997, ASSIGN in 353 msec 2024-11-12T10:55:41,318 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T10:55:41,318 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408941318"}]},"ts":"1731408941318"} 2024-11-12T10:55:41,320 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T10:55:41,321 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T10:55:41,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-11-12T10:55:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-12T10:55:42,288 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:42,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72417d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,288 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:42,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:42,289 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e8a8047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:42,291 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f7c438, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,292 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:42,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:42,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:42,293 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:42,294 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T10:55:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is aed202db7571,45901,1731408812240 2024-11-12T10:55:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3231341a 2024-11-12T10:55:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T10:55:42,295 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T10:55:42,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,297 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:42,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:42,297 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b50eba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:42,299 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f70bc82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:42,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:42,301 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4986d646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:42,302 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75041c09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:42,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:42,306 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fdee10d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:42,308 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:42,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b5b1898, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:42,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:42,310 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:42,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:42,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:42,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed99e5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:42,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:42,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:42,311 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d99dcce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:42,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:42,313 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4144b0c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:42,314 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1186b1f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,315 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:42,315 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:42,316 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:42,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:42,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:42,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f941da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:42,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:42,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:42,317 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:42,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d29984b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:42,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:42,319 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3f33f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:42,320 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc44445, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:42,322 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:42,322 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@454b1e5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:42,324 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:42,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b7e9ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:42,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:42,326 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb13e3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:42,327 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d6e4b2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d70caa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,328 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:55:42,328 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:55:42,329 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:55:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:55:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:55:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5203d100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:55:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:55:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:55:42,330 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:55:42,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e54e04b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:55:42,331 DEBUG [Thread-1915 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:42,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:42,331 DEBUG [Thread-1928 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:42,331 DEBUG [Thread-1923 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:42,332 DEBUG [Thread-1932 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:42,332 DEBUG [Thread-1923 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:42,332 DEBUG [Thread-1923 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-12T10:55:42,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:42,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:42,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:42,334 DEBUG [Thread-1932 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:42,334 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:42,334 DEBUG [Thread-1919 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:42,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:42,335 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:42,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:55:42,336 DEBUG [Thread-1921 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:55:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:55:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:55:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T10:55:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:42,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:55:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:42,342 DEBUG [Thread-1930 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:42,343 DEBUG [Thread-1934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:42,344 DEBUG [Thread-1917 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:42,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:42,345 DEBUG [Thread-1926 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:55:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:55:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:55:42,346 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:42,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:42,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:42,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:42,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:55:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:42,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T10:55:42,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:42,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:42,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:42,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:42,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:42,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:55:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:55:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:55:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:55:42,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2] 2024-11-12T10:55:42,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/bd9219b3688f4dc88287ec74b938c90e is 50, key is test_row_0/A:col10/1731408942345/Put/seqid=0 2024-11-12T10:55:42,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742248_1424 (size=14341) 2024-11-12T10:55:42,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409002391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409002399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408942387, max timestamp=1731408942401Num uniques 859; 2024-11-12T10:55:42,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408942387, max timestamp=1731408942401Num uniques 859; 2024-11-12T10:55:42,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408942387, max timestamp=1731408942401Num uniques 859; 2024-11-12T10:55:42,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:55:42,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:55:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:55:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T10:55:42,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409002443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409002444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409002447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T10:55:42,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:42,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:42,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:42,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:42,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:42,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409002507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409002507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409002557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409002557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409002567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T10:55:42,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:42,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:42,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:42,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:42,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T10:55:42,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409002716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409002717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409002768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409002768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409002777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:42,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/bd9219b3688f4dc88287ec74b938c90e 2024-11-12T10:55:42,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T10:55:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:42,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:42,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/15c0a655fe014ad68b43a2147187aee2 is 50, key is test_row_0/B:col10/1731408942345/Put/seqid=0 2024-11-12T10:55:42,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742249_1425 (size=12001) 2024-11-12T10:55:42,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/15c0a655fe014ad68b43a2147187aee2 2024-11-12T10:55:42,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/9a0ae2aee19f4634837d7106730380dc is 50, key is test_row_0/C:col10/1731408942345/Put/seqid=0 2024-11-12T10:55:42,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742250_1426 (size=12001) 2024-11-12T10:55:42,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/9a0ae2aee19f4634837d7106730380dc 2024-11-12T10:55:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/bd9219b3688f4dc88287ec74b938c90e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/bd9219b3688f4dc88287ec74b938c90e 2024-11-12T10:55:42,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/bd9219b3688f4dc88287ec74b938c90e, entries=200, sequenceid=12, filesize=14.0 K 2024-11-12T10:55:42,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/15c0a655fe014ad68b43a2147187aee2 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/15c0a655fe014ad68b43a2147187aee2 2024-11-12T10:55:42,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/15c0a655fe014ad68b43a2147187aee2, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T10:55:42,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/9a0ae2aee19f4634837d7106730380dc as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9a0ae2aee19f4634837d7106730380dc 2024-11-12T10:55:42,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9a0ae2aee19f4634837d7106730380dc, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T10:55:42,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5b5c37b8175d8b0adea03f092f215997 in 597ms, sequenceid=12, compaction requested=false 2024-11-12T10:55:42,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:42,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T10:55:42,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:42,951 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-12T10:55:42,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:42,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:55:42,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:42,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:55:42,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:42,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:55:42,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/6c481e680fc94ce5bd96ce81d936077d is 50, key is test_row_0/A:col10/1731408942441/Put/seqid=0 2024-11-12T10:55:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T10:55:42,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742251_1427 (size=12001) 2024-11-12T10:55:42,980 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/6c481e680fc94ce5bd96ce81d936077d 2024-11-12T10:55:42,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/e7ce41c1380e4a868c2d207e4a5a3d44 is 50, key is test_row_0/B:col10/1731408942441/Put/seqid=0 2024-11-12T10:55:43,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742252_1428 (size=12001) 2024-11-12T10:55:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:43,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:43,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409003063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:43,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409003068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:43,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:43,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409003088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:43,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409003088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:43,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409003088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:43,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409003177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:43,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409003187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:43,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409003387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:43,401 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/e7ce41c1380e4a868c2d207e4a5a3d44 2024-11-12T10:55:43,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:43,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409003397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:43,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/c2c9e0aa73b84d4a8a7b563eb063046d is 50, key is test_row_0/C:col10/1731408942441/Put/seqid=0 2024-11-12T10:55:43,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742253_1429 (size=12001) 2024-11-12T10:55:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T10:55:43,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409003597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:43,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409003597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:43,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:43,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409003607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:43,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409003697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:43,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409003717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:43,815 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/c2c9e0aa73b84d4a8a7b563eb063046d 2024-11-12T10:55:43,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/6c481e680fc94ce5bd96ce81d936077d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6c481e680fc94ce5bd96ce81d936077d 2024-11-12T10:55:43,820 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6c481e680fc94ce5bd96ce81d936077d, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T10:55:43,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/e7ce41c1380e4a868c2d207e4a5a3d44 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/e7ce41c1380e4a868c2d207e4a5a3d44 2024-11-12T10:55:43,824 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/e7ce41c1380e4a868c2d207e4a5a3d44, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T10:55:43,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/c2c9e0aa73b84d4a8a7b563eb063046d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c2c9e0aa73b84d4a8a7b563eb063046d 2024-11-12T10:55:43,827 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c2c9e0aa73b84d4a8a7b563eb063046d, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T10:55:43,828 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 5b5c37b8175d8b0adea03f092f215997 in 877ms, sequenceid=48, compaction requested=false 2024-11-12T10:55:43,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:43,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:43,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-12T10:55:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-12T10:55:43,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-12T10:55:43,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4940 sec 2024-11-12T10:55:43,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.5000 sec 2024-11-12T10:55:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:44,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T10:55:44,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:44,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:44,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:44,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:44,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:44,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:44,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/d0e03d3562fd4b3282300fe602bfb444 is 50, key is test_row_0/A:col10/1731408944210/Put/seqid=0 2024-11-12T10:55:44,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742254_1430 (size=11997) 2024-11-12T10:55:44,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409004268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:44,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409004270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:44,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409004377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:44,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409004386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T10:55:44,487 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:44,488 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-12T10:55:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T10:55:44,490 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:44,490 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:44,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:44,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409004587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T10:55:44,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409004607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:44,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/d0e03d3562fd4b3282300fe602bfb444 2024-11-12T10:55:44,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409004616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:44,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409004616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:44,623 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in 5b5c37b8175d8b0adea03f092f215997 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,623 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in 5b5c37b8175d8b0adea03f092f215997 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:44,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/8de64ab9053a472fbdbe7e5fdd58f9b4 is 50, key is test_row_0/B:col10/1731408944210/Put/seqid=0 2024-11-12T10:55:44,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742255_1431 (size=9657) 2024-11-12T10:55:44,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409004627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:44,633 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in 5b5c37b8175d8b0adea03f092f215997 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:44,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T10:55:44,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:44,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:44,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:44,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:44,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:44,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T10:55:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:44,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T10:55:44,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409004897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:44,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:44,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:44,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409004917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:44,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:44,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:44,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T10:55:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:44,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:44,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:44,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:45,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/8de64ab9053a472fbdbe7e5fdd58f9b4 2024-11-12T10:55:45,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/ec49d3c719c045f9bdb6ff75c74bd50b is 50, key is test_row_0/C:col10/1731408944210/Put/seqid=0 2024-11-12T10:55:45,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742256_1432 (size=9657) 2024-11-12T10:55:45,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/ec49d3c719c045f9bdb6ff75c74bd50b 2024-11-12T10:55:45,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/d0e03d3562fd4b3282300fe602bfb444 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/d0e03d3562fd4b3282300fe602bfb444 2024-11-12T10:55:45,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/d0e03d3562fd4b3282300fe602bfb444, entries=150, sequenceid=60, filesize=11.7 K 2024-11-12T10:55:45,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/8de64ab9053a472fbdbe7e5fdd58f9b4 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/8de64ab9053a472fbdbe7e5fdd58f9b4 2024-11-12T10:55:45,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/8de64ab9053a472fbdbe7e5fdd58f9b4, entries=100, sequenceid=60, filesize=9.4 K 2024-11-12T10:55:45,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/ec49d3c719c045f9bdb6ff75c74bd50b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/ec49d3c719c045f9bdb6ff75c74bd50b 2024-11-12T10:55:45,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/ec49d3c719c045f9bdb6ff75c74bd50b, entries=100, sequenceid=60, filesize=9.4 K 2024-11-12T10:55:45,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5b5c37b8175d8b0adea03f092f215997 in 859ms, sequenceid=60, compaction requested=true 2024-11-12T10:55:45,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:45,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:55:45,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:45,071 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:45,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:55:45,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:45,071 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:45,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:55:45,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:45,072 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:45,072 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/A is initiating minor compaction (all files) 2024-11-12T10:55:45,072 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/A in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:45,072 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/bd9219b3688f4dc88287ec74b938c90e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6c481e680fc94ce5bd96ce81d936077d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/d0e03d3562fd4b3282300fe602bfb444] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=37.4 K 2024-11-12T10:55:45,072 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:45,072 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/B is initiating minor compaction (all files) 2024-11-12T10:55:45,072 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd9219b3688f4dc88287ec74b938c90e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731408942338 2024-11-12T10:55:45,072 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/B in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:45,072 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/15c0a655fe014ad68b43a2147187aee2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/e7ce41c1380e4a868c2d207e4a5a3d44, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/8de64ab9053a472fbdbe7e5fdd58f9b4] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=32.9 K 2024-11-12T10:55:45,076 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c481e680fc94ce5bd96ce81d936077d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731408942441 2024-11-12T10:55:45,076 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c0a655fe014ad68b43a2147187aee2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731408942344 2024-11-12T10:55:45,076 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0e03d3562fd4b3282300fe602bfb444, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731408943062 2024-11-12T10:55:45,076 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting e7ce41c1380e4a868c2d207e4a5a3d44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731408942441 2024-11-12T10:55:45,076 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 8de64ab9053a472fbdbe7e5fdd58f9b4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731408943065 2024-11-12T10:55:45,086 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#B#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:45,086 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/523b02e7e8b64901949d2888aeb836e0 is 50, key is test_row_0/B:col10/1731408944210/Put/seqid=0 2024-11-12T10:55:45,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T10:55:45,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:45,102 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T10:55:45,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:45,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:45,103 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#A#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:45,103 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/6592621145af4087ba24f76cbedcc2a1 is 50, key is test_row_0/A:col10/1731408944210/Put/seqid=0 2024-11-12T10:55:45,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742257_1433 (size=12104) 2024-11-12T10:55:45,108 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/523b02e7e8b64901949d2888aeb836e0 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/523b02e7e8b64901949d2888aeb836e0 2024-11-12T10:55:45,112 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/B of 5b5c37b8175d8b0adea03f092f215997 into 523b02e7e8b64901949d2888aeb836e0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:45,112 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:45,112 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/B, priority=13, startTime=1731408945071; duration=0sec 2024-11-12T10:55:45,112 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:45,112 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:B 2024-11-12T10:55:45,112 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:45,112 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:45,113 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/C is initiating minor compaction (all files) 2024-11-12T10:55:45,113 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/C in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:45,113 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9a0ae2aee19f4634837d7106730380dc, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c2c9e0aa73b84d4a8a7b563eb063046d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/ec49d3c719c045f9bdb6ff75c74bd50b] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=32.9 K 2024-11-12T10:55:45,113 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a0ae2aee19f4634837d7106730380dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731408942344 2024-11-12T10:55:45,113 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting c2c9e0aa73b84d4a8a7b563eb063046d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731408942441 2024-11-12T10:55:45,114 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ec49d3c719c045f9bdb6ff75c74bd50b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731408943065 2024-11-12T10:55:45,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/cde43de0b2a1473696f634ed8b73e387 is 50, key is test_row_0/A:col10/1731408944268/Put/seqid=0 2024-11-12T10:55:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T10:55:45,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742258_1434 (size=12104) 2024-11-12T10:55:45,129 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#C#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:45,129 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/0ed063df3bf5431ca58fe4b5052c518c is 50, key is test_row_0/C:col10/1731408944210/Put/seqid=0 2024-11-12T10:55:45,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742259_1435 (size=12001) 2024-11-12T10:55:45,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742260_1436 (size=12104) 2024-11-12T10:55:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:45,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:45,420 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T10:55:45,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409005448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:45,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409005460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:45,529 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/6592621145af4087ba24f76cbedcc2a1 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6592621145af4087ba24f76cbedcc2a1 2024-11-12T10:55:45,536 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/cde43de0b2a1473696f634ed8b73e387 2024-11-12T10:55:45,540 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/A of 5b5c37b8175d8b0adea03f092f215997 into 6592621145af4087ba24f76cbedcc2a1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:45,540 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:45,540 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/A, priority=13, startTime=1731408945071; duration=0sec 2024-11-12T10:55:45,540 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:45,541 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:A 2024-11-12T10:55:45,559 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/0ed063df3bf5431ca58fe4b5052c518c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/0ed063df3bf5431ca58fe4b5052c518c 2024-11-12T10:55:45,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/c8f15cac3cb04f9f822a294d47e51d6a is 50, key is test_row_0/B:col10/1731408944268/Put/seqid=0 2024-11-12T10:55:45,566 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/C of 5b5c37b8175d8b0adea03f092f215997 into 0ed063df3bf5431ca58fe4b5052c518c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:45,566 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:45,566 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/C, priority=13, startTime=1731408945071; duration=0sec 2024-11-12T10:55:45,566 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:45,566 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:C 2024-11-12T10:55:45,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409005567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:45,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:45,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:45,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:45,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409005578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:45,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742261_1437 (size=12001) 2024-11-12T10:55:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T10:55:45,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409005787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:45,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409005797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:45,995 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/c8f15cac3cb04f9f822a294d47e51d6a 2024-11-12T10:55:46,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/e1f1ea4fb937441fa031a748a20eb7cb is 50, key is test_row_0/C:col10/1731408944268/Put/seqid=0 2024-11-12T10:55:46,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742262_1438 (size=12001) 2024-11-12T10:55:46,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409006107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:46,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409006117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:46,403 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/e1f1ea4fb937441fa031a748a20eb7cb 2024-11-12T10:55:46,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/cde43de0b2a1473696f634ed8b73e387 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/cde43de0b2a1473696f634ed8b73e387 2024-11-12T10:55:46,418 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/cde43de0b2a1473696f634ed8b73e387, entries=150, sequenceid=80, filesize=11.7 K 2024-11-12T10:55:46,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/c8f15cac3cb04f9f822a294d47e51d6a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c8f15cac3cb04f9f822a294d47e51d6a 2024-11-12T10:55:46,422 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c8f15cac3cb04f9f822a294d47e51d6a, entries=150, sequenceid=80, filesize=11.7 K 2024-11-12T10:55:46,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/e1f1ea4fb937441fa031a748a20eb7cb as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/e1f1ea4fb937441fa031a748a20eb7cb 2024-11-12T10:55:46,429 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/e1f1ea4fb937441fa031a748a20eb7cb, entries=150, sequenceid=80, filesize=11.7 K 2024-11-12T10:55:46,430 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5b5c37b8175d8b0adea03f092f215997 in 1328ms, sequenceid=80, compaction requested=false 2024-11-12T10:55:46,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:46,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:46,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-12T10:55:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-12T10:55:46,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-12T10:55:46,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9410 sec 2024-11-12T10:55:46,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.9460 sec 2024-11-12T10:55:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:46,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T10:55:46,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:46,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:46,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:46,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:46,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:46,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T10:55:46,638 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:46,639 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-12T10:55:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T10:55:46,640 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:46,641 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:46,641 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:46,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/ddaa075745824ce4b001ba762e9c81cf is 50, key is test_row_0/A:col10/1731408946631/Put/seqid=0 2024-11-12T10:55:46,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742263_1439 (size=14341) 2024-11-12T10:55:46,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/ddaa075745824ce4b001ba762e9c81cf 2024-11-12T10:55:46,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/007dab79063c465cb424ae6bdba1ccae is 50, key is test_row_0/B:col10/1731408946631/Put/seqid=0 2024-11-12T10:55:46,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742264_1440 (size=12001) 2024-11-12T10:55:46,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409006683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:46,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409006683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:46,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:46,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409006685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:46,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:46,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409006687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:46,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:46,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:46,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409006694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T10:55:46,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T10:55:46,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:46,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:46,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:46,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:46,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:46,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409006807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:46,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409006807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:46,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409006807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409006807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:46,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:46,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409006817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:46,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T10:55:46,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:46,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:46,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:46,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:46,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T10:55:47,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409007027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409007027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409007027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409007028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409007037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/007dab79063c465cb424ae6bdba1ccae 2024-11-12T10:55:47,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/7321772a465c4cc6ae3d23a6d6b5a9ce is 50, key is test_row_0/C:col10/1731408946631/Put/seqid=0 2024-11-12T10:55:47,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742265_1441 (size=12001) 2024-11-12T10:55:47,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T10:55:47,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:47,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:47,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:47,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:47,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:47,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T10:55:47,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:47,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:47,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:47,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:47,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T10:55:47,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409007336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409007336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409007337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409007347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409007347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T10:55:47,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:47,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:47,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:47,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:47,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:47,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/7321772a465c4cc6ae3d23a6d6b5a9ce 2024-11-12T10:55:47,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/ddaa075745824ce4b001ba762e9c81cf as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/ddaa075745824ce4b001ba762e9c81cf 2024-11-12T10:55:47,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/ddaa075745824ce4b001ba762e9c81cf, entries=200, sequenceid=96, filesize=14.0 K 2024-11-12T10:55:47,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/007dab79063c465cb424ae6bdba1ccae as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/007dab79063c465cb424ae6bdba1ccae 2024-11-12T10:55:47,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/007dab79063c465cb424ae6bdba1ccae, entries=150, sequenceid=96, filesize=11.7 K 2024-11-12T10:55:47,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/7321772a465c4cc6ae3d23a6d6b5a9ce as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/7321772a465c4cc6ae3d23a6d6b5a9ce 2024-11-12T10:55:47,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/7321772a465c4cc6ae3d23a6d6b5a9ce, entries=150, sequenceid=96, filesize=11.7 K 2024-11-12T10:55:47,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5b5c37b8175d8b0adea03f092f215997 in 881ms, sequenceid=96, compaction requested=true 2024-11-12T10:55:47,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:47,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:55:47,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:47,514 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:47,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:55:47,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:47,514 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:47,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:55:47,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:47,515 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:47,515 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:47,515 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/A is initiating minor compaction (all files) 2024-11-12T10:55:47,515 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/B is initiating minor compaction (all files) 2024-11-12T10:55:47,515 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/A in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:47,515 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/B in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:47,515 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6592621145af4087ba24f76cbedcc2a1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/cde43de0b2a1473696f634ed8b73e387, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/ddaa075745824ce4b001ba762e9c81cf] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=37.5 K 2024-11-12T10:55:47,515 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/523b02e7e8b64901949d2888aeb836e0, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c8f15cac3cb04f9f822a294d47e51d6a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/007dab79063c465cb424ae6bdba1ccae] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=35.3 K 2024-11-12T10:55:47,516 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 523b02e7e8b64901949d2888aeb836e0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731408942445 2024-11-12T10:55:47,516 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6592621145af4087ba24f76cbedcc2a1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731408942445 2024-11-12T10:55:47,516 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f15cac3cb04f9f822a294d47e51d6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731408944263 2024-11-12T10:55:47,516 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting cde43de0b2a1473696f634ed8b73e387, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731408944263 2024-11-12T10:55:47,516 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 007dab79063c465cb424ae6bdba1ccae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731408945446 2024-11-12T10:55:47,516 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddaa075745824ce4b001ba762e9c81cf, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731408945434 2024-11-12T10:55:47,523 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#B#compaction#374 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:47,523 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/fb1249c3fb0e4f08b48b3007d001bf28 is 50, key is test_row_0/B:col10/1731408946631/Put/seqid=0 2024-11-12T10:55:47,526 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:47,527 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/132a481bd2b1419f8cc3dbca9661e07b is 50, key is test_row_0/A:col10/1731408946631/Put/seqid=0 2024-11-12T10:55:47,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742266_1442 (size=12207) 2024-11-12T10:55:47,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742267_1443 (size=12207) 2024-11-12T10:55:47,543 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/132a481bd2b1419f8cc3dbca9661e07b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/132a481bd2b1419f8cc3dbca9661e07b 2024-11-12T10:55:47,549 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/A of 5b5c37b8175d8b0adea03f092f215997 into 132a481bd2b1419f8cc3dbca9661e07b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:47,549 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:47,549 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/A, priority=13, startTime=1731408947514; duration=0sec 2024-11-12T10:55:47,549 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:47,549 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:A 2024-11-12T10:55:47,549 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:47,550 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:47,550 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/C is initiating minor compaction (all files) 2024-11-12T10:55:47,550 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/C in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:47,550 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/0ed063df3bf5431ca58fe4b5052c518c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/e1f1ea4fb937441fa031a748a20eb7cb, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/7321772a465c4cc6ae3d23a6d6b5a9ce] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=35.3 K 2024-11-12T10:55:47,550 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ed063df3bf5431ca58fe4b5052c518c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731408942445 2024-11-12T10:55:47,551 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1f1ea4fb937441fa031a748a20eb7cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731408944263 2024-11-12T10:55:47,551 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7321772a465c4cc6ae3d23a6d6b5a9ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731408945446 2024-11-12T10:55:47,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T10:55:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:47,557 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T10:55:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:47,558 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#C#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:47,559 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/74fd77ce97db4bbb9235a9a2e0ab203e is 50, key is test_row_0/C:col10/1731408946631/Put/seqid=0 2024-11-12T10:55:47,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/4c51d67c31f54e66bf71f96ca2c5b3de is 50, key is test_row_0/A:col10/1731408946685/Put/seqid=0 2024-11-12T10:55:47,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742268_1444 (size=12207) 2024-11-12T10:55:47,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742269_1445 (size=12001) 2024-11-12T10:55:47,574 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/4c51d67c31f54e66bf71f96ca2c5b3de 2024-11-12T10:55:47,577 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/74fd77ce97db4bbb9235a9a2e0ab203e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/74fd77ce97db4bbb9235a9a2e0ab203e 2024-11-12T10:55:47,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/55a7262951344379b7baec9626766e94 is 50, key is test_row_0/B:col10/1731408946685/Put/seqid=0 2024-11-12T10:55:47,583 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/C of 5b5c37b8175d8b0adea03f092f215997 into 74fd77ce97db4bbb9235a9a2e0ab203e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:47,583 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:47,583 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/C, priority=13, startTime=1731408947514; duration=0sec 2024-11-12T10:55:47,583 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:47,583 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:C 2024-11-12T10:55:47,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742270_1446 (size=12001) 2024-11-12T10:55:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T10:55:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:47,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:47,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409007877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409007878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409007879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409007881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409007882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,934 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/fb1249c3fb0e4f08b48b3007d001bf28 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/fb1249c3fb0e4f08b48b3007d001bf28 2024-11-12T10:55:47,939 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/B of 5b5c37b8175d8b0adea03f092f215997 into fb1249c3fb0e4f08b48b3007d001bf28(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:47,939 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:47,939 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/B, priority=13, startTime=1731408947514; duration=0sec 2024-11-12T10:55:47,939 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:47,939 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:B 2024-11-12T10:55:47,986 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/55a7262951344379b7baec9626766e94 2024-11-12T10:55:47,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409007987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409007987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/09d15d4e7cff4263b4fc8eb542f83618 is 50, key is test_row_0/C:col10/1731408946685/Put/seqid=0 2024-11-12T10:55:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:47,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742271_1447 (size=12001) 2024-11-12T10:55:48,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409007997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409007997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409007997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409008197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409008197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409008207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409008207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409008207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,396 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/09d15d4e7cff4263b4fc8eb542f83618 2024-11-12T10:55:48,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/4c51d67c31f54e66bf71f96ca2c5b3de as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/4c51d67c31f54e66bf71f96ca2c5b3de 2024-11-12T10:55:48,403 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/4c51d67c31f54e66bf71f96ca2c5b3de, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T10:55:48,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/55a7262951344379b7baec9626766e94 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/55a7262951344379b7baec9626766e94 2024-11-12T10:55:48,406 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/55a7262951344379b7baec9626766e94, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T10:55:48,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/09d15d4e7cff4263b4fc8eb542f83618 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/09d15d4e7cff4263b4fc8eb542f83618 2024-11-12T10:55:48,409 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/09d15d4e7cff4263b4fc8eb542f83618, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T10:55:48,410 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5b5c37b8175d8b0adea03f092f215997 in 853ms, sequenceid=116, compaction requested=false 2024-11-12T10:55:48,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:48,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:48,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-12T10:55:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-12T10:55:48,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-12T10:55:48,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7700 sec 2024-11-12T10:55:48,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.7740 sec 2024-11-12T10:55:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:48,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T10:55:48,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:48,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:48,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:48,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:48,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:48,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:48,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/9825e90f1b3b4bcca47e24ac20a429c4 is 50, key is test_row_0/A:col10/1731408947879/Put/seqid=0 2024-11-12T10:55:48,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742272_1448 (size=14441) 2024-11-12T10:55:48,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409008536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409008537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409008543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409008544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409008545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409008657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409008657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409008657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409008657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409008657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T10:55:48,787 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:48,787 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-12T10:55:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T10:55:48,788 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:48,789 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:48,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:48,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409008867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409008867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409008867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409008867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409008868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T10:55:48,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/9825e90f1b3b4bcca47e24ac20a429c4 2024-11-12T10:55:48,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/c0307e0817994bba8abbf6b5db127eaa is 50, key is test_row_0/B:col10/1731408947879/Put/seqid=0 2024-11-12T10:55:48,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742273_1449 (size=12101) 2024-11-12T10:55:48,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T10:55:48,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:48,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:48,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:48,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:48,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:49,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T10:55:49,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:49,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:49,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:49,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:49,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T10:55:49,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409009177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409009177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409009177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409009187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409009187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T10:55:49,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:49,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:49,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:49,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:49,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:49,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:49,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/c0307e0817994bba8abbf6b5db127eaa 2024-11-12T10:55:49,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/9b5bd5f1079541db84bf9caa84f5fb59 is 50, key is test_row_0/C:col10/1731408947879/Put/seqid=0 2024-11-12T10:55:49,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742274_1450 (size=12101) 2024-11-12T10:55:49,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/9b5bd5f1079541db84bf9caa84f5fb59 2024-11-12T10:55:49,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/9825e90f1b3b4bcca47e24ac20a429c4 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9825e90f1b3b4bcca47e24ac20a429c4 2024-11-12T10:55:49,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9825e90f1b3b4bcca47e24ac20a429c4, entries=200, sequenceid=132, filesize=14.1 K 2024-11-12T10:55:49,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/c0307e0817994bba8abbf6b5db127eaa as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c0307e0817994bba8abbf6b5db127eaa 2024-11-12T10:55:49,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c0307e0817994bba8abbf6b5db127eaa, entries=150, sequenceid=132, filesize=11.8 K 2024-11-12T10:55:49,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/9b5bd5f1079541db84bf9caa84f5fb59 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9b5bd5f1079541db84bf9caa84f5fb59 2024-11-12T10:55:49,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9b5bd5f1079541db84bf9caa84f5fb59, entries=150, sequenceid=132, filesize=11.8 K 2024-11-12T10:55:49,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5b5c37b8175d8b0adea03f092f215997 in 853ms, sequenceid=132, compaction requested=true 2024-11-12T10:55:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:55:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:49,366 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:55:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:49,366 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:55:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:49,367 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:49,367 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/A is initiating minor compaction (all files) 2024-11-12T10:55:49,367 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/A in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:49,367 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/132a481bd2b1419f8cc3dbca9661e07b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/4c51d67c31f54e66bf71f96ca2c5b3de, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9825e90f1b3b4bcca47e24ac20a429c4] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=37.7 K 2024-11-12T10:55:49,369 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:49,369 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/B is initiating minor compaction (all files) 2024-11-12T10:55:49,369 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/B in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:49,369 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/fb1249c3fb0e4f08b48b3007d001bf28, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/55a7262951344379b7baec9626766e94, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c0307e0817994bba8abbf6b5db127eaa] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=35.5 K 2024-11-12T10:55:49,369 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 132a481bd2b1419f8cc3dbca9661e07b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731408945446 2024-11-12T10:55:49,369 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting fb1249c3fb0e4f08b48b3007d001bf28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731408945446 2024-11-12T10:55:49,369 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c51d67c31f54e66bf71f96ca2c5b3de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731408946680 2024-11-12T10:55:49,369 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 55a7262951344379b7baec9626766e94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731408946680 2024-11-12T10:55:49,370 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting c0307e0817994bba8abbf6b5db127eaa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731408947879 2024-11-12T10:55:49,370 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9825e90f1b3b4bcca47e24ac20a429c4, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731408947878 2024-11-12T10:55:49,383 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#B#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:49,384 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/5e03d7d6779b4b798c8c3ad75bbc6962 is 50, key is test_row_0/B:col10/1731408947879/Put/seqid=0 2024-11-12T10:55:49,387 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#A#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:49,387 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/71d6d4aff837405fb7fb65dda8e69d57 is 50, key is test_row_0/A:col10/1731408947879/Put/seqid=0 2024-11-12T10:55:49,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T10:55:49,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:49,400 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T10:55:49,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:49,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:49,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:49,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:49,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:49,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:49,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742275_1451 (size=12409) 2024-11-12T10:55:49,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/834b7c2ac6ed445c9d785f72bfecb664 is 50, key is test_row_0/A:col10/1731408948544/Put/seqid=0 2024-11-12T10:55:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T10:55:49,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742276_1452 (size=12409) 2024-11-12T10:55:49,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742277_1453 (size=12151) 2024-11-12T10:55:49,436 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/834b7c2ac6ed445c9d785f72bfecb664 2024-11-12T10:55:49,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/64f1b6d3e8fa4f5a805196d2457ba499 is 50, key is test_row_0/B:col10/1731408948544/Put/seqid=0 2024-11-12T10:55:49,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742278_1454 (size=12151) 2024-11-12T10:55:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:49,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:49,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409009714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409009715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409009715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409009715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409009720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,815 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/5e03d7d6779b4b798c8c3ad75bbc6962 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/5e03d7d6779b4b798c8c3ad75bbc6962 2024-11-12T10:55:49,819 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/B of 5b5c37b8175d8b0adea03f092f215997 into 5e03d7d6779b4b798c8c3ad75bbc6962(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:49,819 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:49,819 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/B, priority=13, startTime=1731408949366; duration=0sec 2024-11-12T10:55:49,819 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:49,819 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:B 2024-11-12T10:55:49,819 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:49,820 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:49,820 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/C is initiating minor compaction (all files) 2024-11-12T10:55:49,820 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/C in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:49,820 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/74fd77ce97db4bbb9235a9a2e0ab203e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/09d15d4e7cff4263b4fc8eb542f83618, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9b5bd5f1079541db84bf9caa84f5fb59] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=35.5 K 2024-11-12T10:55:49,820 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 74fd77ce97db4bbb9235a9a2e0ab203e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731408945446 2024-11-12T10:55:49,821 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 09d15d4e7cff4263b4fc8eb542f83618, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731408946680 2024-11-12T10:55:49,821 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b5bd5f1079541db84bf9caa84f5fb59, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731408947879 2024-11-12T10:55:49,827 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#C#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:49,827 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/d807cd07d90b4554bc1a22e7e7589a16 is 50, key is test_row_0/C:col10/1731408947879/Put/seqid=0 2024-11-12T10:55:49,827 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/71d6d4aff837405fb7fb65dda8e69d57 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/71d6d4aff837405fb7fb65dda8e69d57 2024-11-12T10:55:49,832 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/A of 5b5c37b8175d8b0adea03f092f215997 into 71d6d4aff837405fb7fb65dda8e69d57(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:49,832 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:49,832 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/A, priority=13, startTime=1731408949366; duration=0sec 2024-11-12T10:55:49,832 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:49,832 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:A 2024-11-12T10:55:49,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742279_1455 (size=12409) 2024-11-12T10:55:49,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409009827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409009837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409009837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409009837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409009837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:49,877 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/64f1b6d3e8fa4f5a805196d2457ba499 2024-11-12T10:55:49,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/19808052c8684f67968cbedf9ce573d6 is 50, key is test_row_0/C:col10/1731408948544/Put/seqid=0 2024-11-12T10:55:49,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742280_1456 (size=12151) 2024-11-12T10:55:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T10:55:50,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409010037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409010057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409010057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409010058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409010058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,237 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/d807cd07d90b4554bc1a22e7e7589a16 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/d807cd07d90b4554bc1a22e7e7589a16 2024-11-12T10:55:50,240 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/C of 5b5c37b8175d8b0adea03f092f215997 into d807cd07d90b4554bc1a22e7e7589a16(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:50,240 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:50,240 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/C, priority=13, startTime=1731408949366; duration=0sec 2024-11-12T10:55:50,240 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:50,241 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:C 2024-11-12T10:55:50,287 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/19808052c8684f67968cbedf9ce573d6 2024-11-12T10:55:50,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/834b7c2ac6ed445c9d785f72bfecb664 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/834b7c2ac6ed445c9d785f72bfecb664 2024-11-12T10:55:50,294 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/834b7c2ac6ed445c9d785f72bfecb664, entries=150, sequenceid=150, filesize=11.9 K 2024-11-12T10:55:50,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/64f1b6d3e8fa4f5a805196d2457ba499 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/64f1b6d3e8fa4f5a805196d2457ba499 2024-11-12T10:55:50,306 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/64f1b6d3e8fa4f5a805196d2457ba499, entries=150, sequenceid=150, filesize=11.9 K 2024-11-12T10:55:50,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/19808052c8684f67968cbedf9ce573d6 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/19808052c8684f67968cbedf9ce573d6 2024-11-12T10:55:50,312 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/19808052c8684f67968cbedf9ce573d6, entries=150, sequenceid=150, filesize=11.9 K 2024-11-12T10:55:50,313 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 5b5c37b8175d8b0adea03f092f215997 in 913ms, sequenceid=150, compaction requested=false 2024-11-12T10:55:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-12T10:55:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-12T10:55:50,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-12T10:55:50,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5250 sec 2024-11-12T10:55:50,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.5290 sec 2024-11-12T10:55:50,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:50,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T10:55:50,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:50,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:50,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:50,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:50,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:50,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:50,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/493dc215fc9f447896b48f3a1cd9d019 is 50, key is test_row_0/A:col10/1731408950351/Put/seqid=0 2024-11-12T10:55:50,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742281_1457 (size=14541) 2024-11-12T10:55:50,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409010388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409010393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409010399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409010400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409010400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409010507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409010507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409010507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409010508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409010508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409010727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409010728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409010728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409010728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409010728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:50,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/493dc215fc9f447896b48f3a1cd9d019 2024-11-12T10:55:50,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/5a71f28f82d64037a3328bcdb5427919 is 50, key is test_row_0/B:col10/1731408950351/Put/seqid=0 2024-11-12T10:55:50,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742282_1458 (size=12151) 2024-11-12T10:55:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T10:55:50,937 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:50,938 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-12T10:55:50,940 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T10:55:50,940 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:50,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:51,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409011036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T10:55:51,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409011047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409011047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409011047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:51,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409011048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:51,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T10:55:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:51,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:51,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/5a71f28f82d64037a3328bcdb5427919 2024-11-12T10:55:51,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/c3e6f73aee90431fb068f14485c35841 is 50, key is test_row_0/C:col10/1731408950351/Put/seqid=0 2024-11-12T10:55:51,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742283_1459 (size=12151) 2024-11-12T10:55:51,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T10:55:51,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:51,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:51,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:51,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:51,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T10:55:51,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T10:55:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:51,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:51,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T10:55:51,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:51,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:51,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:51,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:51,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:51,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409011547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:51,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T10:55:51,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409011568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:51,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409011568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:51,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409011568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:51,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409011568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:51,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:51,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:51,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/c3e6f73aee90431fb068f14485c35841 2024-11-12T10:55:51,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/493dc215fc9f447896b48f3a1cd9d019 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/493dc215fc9f447896b48f3a1cd9d019 2024-11-12T10:55:51,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/493dc215fc9f447896b48f3a1cd9d019, entries=200, sequenceid=168, filesize=14.2 K 2024-11-12T10:55:51,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/5a71f28f82d64037a3328bcdb5427919 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/5a71f28f82d64037a3328bcdb5427919 2024-11-12T10:55:51,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/5a71f28f82d64037a3328bcdb5427919, entries=150, sequenceid=168, filesize=11.9 K 2024-11-12T10:55:51,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/c3e6f73aee90431fb068f14485c35841 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c3e6f73aee90431fb068f14485c35841 2024-11-12T10:55:51,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c3e6f73aee90431fb068f14485c35841, entries=150, sequenceid=168, filesize=11.9 K 2024-11-12T10:55:51,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 5b5c37b8175d8b0adea03f092f215997 in 1247ms, sequenceid=168, compaction requested=true 2024-11-12T10:55:51,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:51,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:55:51,599 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:51,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:51,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:55:51,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:51,599 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:51,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:55:51,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:51,600 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:51,600 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:51,600 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/B is initiating minor compaction (all files) 2024-11-12T10:55:51,600 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/A is initiating minor compaction (all files) 2024-11-12T10:55:51,600 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/A in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:51,600 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/B in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:51,600 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/5e03d7d6779b4b798c8c3ad75bbc6962, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/64f1b6d3e8fa4f5a805196d2457ba499, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/5a71f28f82d64037a3328bcdb5427919] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=35.9 K 2024-11-12T10:55:51,600 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/71d6d4aff837405fb7fb65dda8e69d57, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/834b7c2ac6ed445c9d785f72bfecb664, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/493dc215fc9f447896b48f3a1cd9d019] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=38.2 K 2024-11-12T10:55:51,600 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e03d7d6779b4b798c8c3ad75bbc6962, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731408947879 2024-11-12T10:55:51,600 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71d6d4aff837405fb7fb65dda8e69d57, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731408947879 2024-11-12T10:55:51,600 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 64f1b6d3e8fa4f5a805196d2457ba499, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731408948524 2024-11-12T10:55:51,600 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 834b7c2ac6ed445c9d785f72bfecb664, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731408948524 2024-11-12T10:55:51,601 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a71f28f82d64037a3328bcdb5427919, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731408949712 2024-11-12T10:55:51,601 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 493dc215fc9f447896b48f3a1cd9d019, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731408949712 2024-11-12T10:55:51,607 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#B#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:51,607 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:51,607 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/2c228ecf71ac41f4a1f6511b6f240455 is 50, key is test_row_0/B:col10/1731408950351/Put/seqid=0 2024-11-12T10:55:51,607 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/045472d13dc44c328428c0e1bf40814d is 50, key is test_row_0/A:col10/1731408950351/Put/seqid=0 2024-11-12T10:55:51,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742284_1460 (size=12561) 2024-11-12T10:55:51,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742285_1461 (size=12561) 2024-11-12T10:55:51,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T10:55:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:51,706 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T10:55:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:51,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/5586409a4bae42a190d5137e3d16e8b8 is 50, key is test_row_0/A:col10/1731408950387/Put/seqid=0 2024-11-12T10:55:51,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742286_1462 (size=12151) 2024-11-12T10:55:52,018 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/2c228ecf71ac41f4a1f6511b6f240455 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/2c228ecf71ac41f4a1f6511b6f240455 2024-11-12T10:55:52,026 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/B of 5b5c37b8175d8b0adea03f092f215997 into 2c228ecf71ac41f4a1f6511b6f240455(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:52,026 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:52,026 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/B, priority=13, startTime=1731408951599; duration=0sec 2024-11-12T10:55:52,026 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:52,026 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:B 2024-11-12T10:55:52,026 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:52,026 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/045472d13dc44c328428c0e1bf40814d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/045472d13dc44c328428c0e1bf40814d 2024-11-12T10:55:52,027 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:52,027 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/C is initiating minor compaction (all files) 2024-11-12T10:55:52,027 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/C in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:52,027 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/d807cd07d90b4554bc1a22e7e7589a16, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/19808052c8684f67968cbedf9ce573d6, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c3e6f73aee90431fb068f14485c35841] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=35.9 K 2024-11-12T10:55:52,028 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting d807cd07d90b4554bc1a22e7e7589a16, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731408947879 2024-11-12T10:55:52,028 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 19808052c8684f67968cbedf9ce573d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731408948524 2024-11-12T10:55:52,028 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e6f73aee90431fb068f14485c35841, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731408949712 2024-11-12T10:55:52,031 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/A of 5b5c37b8175d8b0adea03f092f215997 into 045472d13dc44c328428c0e1bf40814d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:52,031 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:52,031 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/A, priority=13, startTime=1731408951599; duration=0sec 2024-11-12T10:55:52,031 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:52,031 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:A 2024-11-12T10:55:52,055 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#C#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:52,056 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/682b1a30af1d4ff1a1e20cbe359719ad is 50, key is test_row_0/C:col10/1731408950351/Put/seqid=0 2024-11-12T10:55:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T10:55:52,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742287_1463 (size=12561) 2024-11-12T10:55:52,088 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/682b1a30af1d4ff1a1e20cbe359719ad as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/682b1a30af1d4ff1a1e20cbe359719ad 2024-11-12T10:55:52,094 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/C of 5b5c37b8175d8b0adea03f092f215997 into 682b1a30af1d4ff1a1e20cbe359719ad(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:52,094 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:52,094 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/C, priority=13, startTime=1731408951599; duration=0sec 2024-11-12T10:55:52,094 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:52,094 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:C 2024-11-12T10:55:52,114 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/5586409a4bae42a190d5137e3d16e8b8 2024-11-12T10:55:52,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/9f438e19392c4e66b0145362f729e456 is 50, key is test_row_0/B:col10/1731408950387/Put/seqid=0 2024-11-12T10:55:52,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742288_1464 (size=12151) 2024-11-12T10:55:52,531 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/9f438e19392c4e66b0145362f729e456 2024-11-12T10:55:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/b578285d246045d3b24998d70709be2e is 50, key is test_row_0/C:col10/1731408950387/Put/seqid=0 2024-11-12T10:55:52,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742289_1465 (size=12151) 2024-11-12T10:55:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:52,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409012592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409012592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409012593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409012595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409012596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409012707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409012707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409012707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409012707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409012707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409012917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409012917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409012917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409012917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409012926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:52,949 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/b578285d246045d3b24998d70709be2e 2024-11-12T10:55:52,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/5586409a4bae42a190d5137e3d16e8b8 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/5586409a4bae42a190d5137e3d16e8b8 2024-11-12T10:55:52,957 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/5586409a4bae42a190d5137e3d16e8b8, entries=150, sequenceid=186, filesize=11.9 K 2024-11-12T10:55:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/9f438e19392c4e66b0145362f729e456 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9f438e19392c4e66b0145362f729e456 2024-11-12T10:55:52,962 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9f438e19392c4e66b0145362f729e456, entries=150, sequenceid=186, filesize=11.9 K 2024-11-12T10:55:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/b578285d246045d3b24998d70709be2e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/b578285d246045d3b24998d70709be2e 2024-11-12T10:55:52,966 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/b578285d246045d3b24998d70709be2e, entries=150, sequenceid=186, filesize=11.9 K 2024-11-12T10:55:52,967 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 5b5c37b8175d8b0adea03f092f215997 in 1260ms, sequenceid=186, compaction requested=false 2024-11-12T10:55:52,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:52,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:52,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-12T10:55:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-12T10:55:52,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-12T10:55:52,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0280 sec 2024-11-12T10:55:52,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.0310 sec 2024-11-12T10:55:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T10:55:53,087 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:53,087 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-12T10:55:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T10:55:53,089 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:53,089 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:53,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:53,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T10:55:53,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:53,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T10:55:53,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:53,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:53,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:53,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:53,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:53,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:53,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/641c76a954694ad0886f5c56d62f1241 is 50, key is test_row_0/A:col10/1731408952595/Put/seqid=0 2024-11-12T10:55:53,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742290_1466 (size=14541) 2024-11-12T10:55:53,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T10:55:53,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:53,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:53,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:53,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409013250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409013260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409013261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409013261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409013262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409013367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409013367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409013367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409013377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409013386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T10:55:53,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:53,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:53,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:53,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T10:55:53,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T10:55:53,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:53,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:53,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:53,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409013577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409013577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409013577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409013587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409013597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/641c76a954694ad0886f5c56d62f1241 2024-11-12T10:55:53,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/613da75614d347639f4f40f7dbdde295 is 50, key is test_row_0/B:col10/1731408952595/Put/seqid=0 2024-11-12T10:55:53,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742291_1467 (size=12151) 2024-11-12T10:55:53,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T10:55:53,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:53,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:53,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:53,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T10:55:53,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T10:55:53,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:53,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:53,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:53,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:53,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409013887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409013887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409013896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409013907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:53,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409013917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:54,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T10:55:54,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:54,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:54,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:54,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:54,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:54,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/613da75614d347639f4f40f7dbdde295 2024-11-12T10:55:54,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/8e250ec01d364c6fa814110c9bebc6b6 is 50, key is test_row_0/C:col10/1731408952595/Put/seqid=0 2024-11-12T10:55:54,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742292_1468 (size=12151) 2024-11-12T10:55:54,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T10:55:54,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:54,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:54,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:54,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:54,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T10:55:54,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T10:55:54,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:54,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:54,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:54,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:54,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409014407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:54,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:54,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409014407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:54,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409014407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:54,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409014416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:54,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:54,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:54,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:54,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409014427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:54,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T10:55:54,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:54,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:54,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:54,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:54,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:54,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/8e250ec01d364c6fa814110c9bebc6b6 2024-11-12T10:55:54,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/641c76a954694ad0886f5c56d62f1241 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/641c76a954694ad0886f5c56d62f1241 2024-11-12T10:55:54,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/641c76a954694ad0886f5c56d62f1241, entries=200, sequenceid=204, filesize=14.2 K 2024-11-12T10:55:54,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/613da75614d347639f4f40f7dbdde295 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/613da75614d347639f4f40f7dbdde295 2024-11-12T10:55:54,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/613da75614d347639f4f40f7dbdde295, entries=150, sequenceid=204, filesize=11.9 K 2024-11-12T10:55:54,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/8e250ec01d364c6fa814110c9bebc6b6 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/8e250ec01d364c6fa814110c9bebc6b6 2024-11-12T10:55:54,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/8e250ec01d364c6fa814110c9bebc6b6, entries=150, sequenceid=204, filesize=11.9 K 2024-11-12T10:55:54,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 5b5c37b8175d8b0adea03f092f215997 in 1244ms, sequenceid=204, compaction requested=true 2024-11-12T10:55:54,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:54,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:55:54,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:54,478 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:54,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:55:54,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:54,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:55:54,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:54,478 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:54,479 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:54,479 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:54,479 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/A is initiating minor compaction (all files) 2024-11-12T10:55:54,479 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/B is initiating minor compaction (all files) 2024-11-12T10:55:54,479 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/A in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:54,479 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/B in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:54,479 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/045472d13dc44c328428c0e1bf40814d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/5586409a4bae42a190d5137e3d16e8b8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/641c76a954694ad0886f5c56d62f1241] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=38.3 K 2024-11-12T10:55:54,479 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/2c228ecf71ac41f4a1f6511b6f240455, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9f438e19392c4e66b0145362f729e456, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/613da75614d347639f4f40f7dbdde295] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=36.0 K 2024-11-12T10:55:54,479 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 045472d13dc44c328428c0e1bf40814d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731408949712 2024-11-12T10:55:54,479 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c228ecf71ac41f4a1f6511b6f240455, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731408949712 2024-11-12T10:55:54,479 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5586409a4bae42a190d5137e3d16e8b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731408950387 2024-11-12T10:55:54,480 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f438e19392c4e66b0145362f729e456, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731408950387 2024-11-12T10:55:54,480 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 613da75614d347639f4f40f7dbdde295, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731408952578 2024-11-12T10:55:54,480 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 641c76a954694ad0886f5c56d62f1241, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731408952578 2024-11-12T10:55:54,485 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#B#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:54,485 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#A#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:54,486 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/a6d033d130f542c8b3e355f00fb6ec9b is 50, key is test_row_0/A:col10/1731408952595/Put/seqid=0 2024-11-12T10:55:54,486 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/189441efc45c44c3ad520499f4ef50d8 is 50, key is test_row_0/B:col10/1731408952595/Put/seqid=0 2024-11-12T10:55:54,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742294_1470 (size=12663) 2024-11-12T10:55:54,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742293_1469 (size=12663) 2024-11-12T10:55:54,520 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/a6d033d130f542c8b3e355f00fb6ec9b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/a6d033d130f542c8b3e355f00fb6ec9b 2024-11-12T10:55:54,524 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/189441efc45c44c3ad520499f4ef50d8 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/189441efc45c44c3ad520499f4ef50d8 2024-11-12T10:55:54,527 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/A of 5b5c37b8175d8b0adea03f092f215997 into a6d033d130f542c8b3e355f00fb6ec9b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:54,527 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:54,527 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/A, priority=13, startTime=1731408954478; duration=0sec 2024-11-12T10:55:54,528 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:54,528 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:A 2024-11-12T10:55:54,528 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:54,528 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/B of 5b5c37b8175d8b0adea03f092f215997 into 189441efc45c44c3ad520499f4ef50d8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:54,528 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:54,528 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:54,528 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/B, priority=13, startTime=1731408954478; duration=0sec 2024-11-12T10:55:54,529 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/C is initiating minor compaction (all files) 2024-11-12T10:55:54,529 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:54,529 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:B 2024-11-12T10:55:54,529 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/C in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:54,529 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/682b1a30af1d4ff1a1e20cbe359719ad, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/b578285d246045d3b24998d70709be2e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/8e250ec01d364c6fa814110c9bebc6b6] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=36.0 K 2024-11-12T10:55:54,529 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 682b1a30af1d4ff1a1e20cbe359719ad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731408949712 2024-11-12T10:55:54,530 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting b578285d246045d3b24998d70709be2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731408950387 2024-11-12T10:55:54,531 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e250ec01d364c6fa814110c9bebc6b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731408952578 2024-11-12T10:55:54,536 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#C#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:54,536 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/b492b6bac20e43e38f027026a83c33d4 is 50, key is test_row_0/C:col10/1731408952595/Put/seqid=0 2024-11-12T10:55:54,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742295_1471 (size=12663) 2024-11-12T10:55:54,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T10:55:54,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:54,615 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T10:55:54,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:54,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:54,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:54,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:54,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:54,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:54,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/6ceefd5425fc4d608861b34fb91c3e45 is 50, key is test_row_0/A:col10/1731408953248/Put/seqid=0 2024-11-12T10:55:54,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742296_1472 (size=12151) 2024-11-12T10:55:54,945 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/b492b6bac20e43e38f027026a83c33d4 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/b492b6bac20e43e38f027026a83c33d4 2024-11-12T10:55:54,948 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/C of 5b5c37b8175d8b0adea03f092f215997 into b492b6bac20e43e38f027026a83c33d4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:54,948 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:54,948 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/C, priority=13, startTime=1731408954478; duration=0sec 2024-11-12T10:55:54,948 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:54,948 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:C 2024-11-12T10:55:55,023 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/6ceefd5425fc4d608861b34fb91c3e45 2024-11-12T10:55:55,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/9b74276cfd61441aab59c32793c35cd9 is 50, key is test_row_0/B:col10/1731408953248/Put/seqid=0 2024-11-12T10:55:55,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742297_1473 (size=12151) 2024-11-12T10:55:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T10:55:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:55,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:55,433 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/9b74276cfd61441aab59c32793c35cd9 2024-11-12T10:55:55,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/9388dfeaf52141a79a80e9385337fdb8 is 50, key is test_row_0/C:col10/1731408953248/Put/seqid=0 2024-11-12T10:55:55,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742298_1474 (size=12151) 2024-11-12T10:55:55,446 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/9388dfeaf52141a79a80e9385337fdb8 2024-11-12T10:55:55,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409015442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/6ceefd5425fc4d608861b34fb91c3e45 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6ceefd5425fc4d608861b34fb91c3e45 2024-11-12T10:55:55,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409015443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409015445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409015446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,455 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6ceefd5425fc4d608861b34fb91c3e45, entries=150, sequenceid=223, filesize=11.9 K 2024-11-12T10:55:55,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/9b74276cfd61441aab59c32793c35cd9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9b74276cfd61441aab59c32793c35cd9 2024-11-12T10:55:55,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409015447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,457 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in 5b5c37b8175d8b0adea03f092f215997 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,459 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9b74276cfd61441aab59c32793c35cd9, entries=150, sequenceid=223, filesize=11.9 K 2024-11-12T10:55:55,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/9388dfeaf52141a79a80e9385337fdb8 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9388dfeaf52141a79a80e9385337fdb8 2024-11-12T10:55:55,462 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9388dfeaf52141a79a80e9385337fdb8, entries=150, sequenceid=223, filesize=11.9 K 2024-11-12T10:55:55,462 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 5b5c37b8175d8b0adea03f092f215997 in 847ms, sequenceid=223, compaction requested=false 2024-11-12T10:55:55,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:55,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:55,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-12T10:55:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-12T10:55:55,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-12T10:55:55,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3750 sec 2024-11-12T10:55:55,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.3810 sec 2024-11-12T10:55:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:55,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T10:55:55,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:55,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:55,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:55,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:55,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:55,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:55,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/13a762f8db96498582b33773e4dc92a9 is 50, key is test_row_0/A:col10/1731408955445/Put/seqid=0 2024-11-12T10:55:55,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742299_1475 (size=14541) 2024-11-12T10:55:55,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409015584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409015585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409015586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409015588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409015697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409015706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409015707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409015707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409015906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409015917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409015927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409015927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:55,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/13a762f8db96498582b33773e4dc92a9 2024-11-12T10:55:55,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/10f8de0a262b4268a13c5172dc4419d7 is 50, key is test_row_0/B:col10/1731408955445/Put/seqid=0 2024-11-12T10:55:55,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742300_1476 (size=12151) 2024-11-12T10:55:56,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409016217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:56,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409016227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:56,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409016247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:56,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409016247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:56,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/10f8de0a262b4268a13c5172dc4419d7 2024-11-12T10:55:56,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/4101fcb2ba1b4ad4aaf9f08295979e91 is 50, key is test_row_0/C:col10/1731408955445/Put/seqid=0 2024-11-12T10:55:56,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742301_1477 (size=12151) 2024-11-12T10:55:56,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409016727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:56,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:56,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409016736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:56,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:56,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409016757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:56,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409016757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:56,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/4101fcb2ba1b4ad4aaf9f08295979e91 2024-11-12T10:55:56,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/13a762f8db96498582b33773e4dc92a9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/13a762f8db96498582b33773e4dc92a9 2024-11-12T10:55:56,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/13a762f8db96498582b33773e4dc92a9, entries=200, sequenceid=240, filesize=14.2 K 2024-11-12T10:55:56,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/10f8de0a262b4268a13c5172dc4419d7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/10f8de0a262b4268a13c5172dc4419d7 2024-11-12T10:55:56,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/10f8de0a262b4268a13c5172dc4419d7, entries=150, sequenceid=240, filesize=11.9 K 2024-11-12T10:55:56,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/4101fcb2ba1b4ad4aaf9f08295979e91 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/4101fcb2ba1b4ad4aaf9f08295979e91 2024-11-12T10:55:56,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/4101fcb2ba1b4ad4aaf9f08295979e91, entries=150, sequenceid=240, filesize=11.9 K 2024-11-12T10:55:56,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5b5c37b8175d8b0adea03f092f215997 in 1247ms, sequenceid=240, compaction requested=true 2024-11-12T10:55:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:56,808 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:55:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:56,808 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:56,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:55:56,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:56,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:55:56,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:56,809 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:56,809 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/A is initiating minor compaction (all files) 2024-11-12T10:55:56,809 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/A in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:56,809 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/a6d033d130f542c8b3e355f00fb6ec9b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6ceefd5425fc4d608861b34fb91c3e45, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/13a762f8db96498582b33773e4dc92a9] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=38.4 K 2024-11-12T10:55:56,810 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:56,810 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6d033d130f542c8b3e355f00fb6ec9b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731408952578 2024-11-12T10:55:56,810 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/B is initiating minor compaction (all files) 2024-11-12T10:55:56,810 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/B in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:56,810 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/189441efc45c44c3ad520499f4ef50d8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9b74276cfd61441aab59c32793c35cd9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/10f8de0a262b4268a13c5172dc4419d7] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=36.1 K 2024-11-12T10:55:56,810 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ceefd5425fc4d608861b34fb91c3e45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731408953248 2024-11-12T10:55:56,810 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 189441efc45c44c3ad520499f4ef50d8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731408952578 2024-11-12T10:55:56,810 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13a762f8db96498582b33773e4dc92a9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731408955432 2024-11-12T10:55:56,810 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b74276cfd61441aab59c32793c35cd9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731408953248 2024-11-12T10:55:56,810 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 10f8de0a262b4268a13c5172dc4419d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731408955444 2024-11-12T10:55:56,815 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#B#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:56,815 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/ae6ea0ec8b9a47709031dab65f41b477 is 50, key is test_row_0/B:col10/1731408955445/Put/seqid=0 2024-11-12T10:55:56,815 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#A#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:56,816 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/0a55f49c147a441b88d2dff80a82b172 is 50, key is test_row_0/A:col10/1731408955445/Put/seqid=0 2024-11-12T10:55:56,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742302_1478 (size=12765) 2024-11-12T10:55:56,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742303_1479 (size=12765) 2024-11-12T10:55:57,222 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/ae6ea0ec8b9a47709031dab65f41b477 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/ae6ea0ec8b9a47709031dab65f41b477 2024-11-12T10:55:57,223 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/0a55f49c147a441b88d2dff80a82b172 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/0a55f49c147a441b88d2dff80a82b172 2024-11-12T10:55:57,226 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/B of 5b5c37b8175d8b0adea03f092f215997 into ae6ea0ec8b9a47709031dab65f41b477(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:57,226 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:57,226 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/B, priority=13, startTime=1731408956808; duration=0sec 2024-11-12T10:55:57,226 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:57,226 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:B 2024-11-12T10:55:57,226 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:55:57,226 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/A of 5b5c37b8175d8b0adea03f092f215997 into 0a55f49c147a441b88d2dff80a82b172(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:57,226 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:57,226 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/A, priority=13, startTime=1731408956808; duration=0sec 2024-11-12T10:55:57,226 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:57,226 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:A 2024-11-12T10:55:57,227 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:55:57,227 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/C is initiating minor compaction (all files) 2024-11-12T10:55:57,227 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/C in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:57,227 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/b492b6bac20e43e38f027026a83c33d4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9388dfeaf52141a79a80e9385337fdb8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/4101fcb2ba1b4ad4aaf9f08295979e91] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=36.1 K 2024-11-12T10:55:57,227 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting b492b6bac20e43e38f027026a83c33d4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731408952578 2024-11-12T10:55:57,227 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 9388dfeaf52141a79a80e9385337fdb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731408953248 2024-11-12T10:55:57,227 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 4101fcb2ba1b4ad4aaf9f08295979e91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731408955444 2024-11-12T10:55:57,234 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#C#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:57,234 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/f13d84a32f164effb9f1636e0da6fba2 is 50, key is test_row_0/C:col10/1731408955445/Put/seqid=0 2024-11-12T10:55:57,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742304_1480 (size=12765) 2024-11-12T10:55:57,255 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/f13d84a32f164effb9f1636e0da6fba2 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/f13d84a32f164effb9f1636e0da6fba2 2024-11-12T10:55:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T10:55:57,258 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:57,258 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-12T10:55:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T10:55:57,261 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:57,270 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:57,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:57,270 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/C of 5b5c37b8175d8b0adea03f092f215997 into f13d84a32f164effb9f1636e0da6fba2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:57,270 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:57,270 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/C, priority=13, startTime=1731408956809; duration=0sec 2024-11-12T10:55:57,270 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:57,270 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:C 2024-11-12T10:55:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T10:55:57,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T10:55:57,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:57,423 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T10:55:57,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:57,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:57,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:57,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:57,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:57,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:57,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/9f99e64261de423fb077abb968fe625c is 50, key is test_row_0/A:col10/1731408955585/Put/seqid=0 2024-11-12T10:55:57,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742305_1481 (size=12151) 2024-11-12T10:55:57,431 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/9f99e64261de423fb077abb968fe625c 2024-11-12T10:55:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/74289cb4f9074f7db2e87b397dc78248 is 50, key is test_row_0/B:col10/1731408955585/Put/seqid=0 2024-11-12T10:55:57,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742306_1482 (size=12151) 2024-11-12T10:55:57,443 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/74289cb4f9074f7db2e87b397dc78248 2024-11-12T10:55:57,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/85f3f60804d14eedb1c5ad03a7fb6693 is 50, key is test_row_0/C:col10/1731408955585/Put/seqid=0 2024-11-12T10:55:57,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742307_1483 (size=12151) 2024-11-12T10:55:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:57,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:57,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409017541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T10:55:57,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409017657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:57,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:57,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409017747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:57,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:57,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409017747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:57,754 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 5b5c37b8175d8b0adea03f092f215997 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:57,755 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in 5b5c37b8175d8b0adea03f092f215997 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:57,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409017777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:57,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409017777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:57,784 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in 5b5c37b8175d8b0adea03f092f215997 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:57,785 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 5b5c37b8175d8b0adea03f092f215997 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:57,866 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/85f3f60804d14eedb1c5ad03a7fb6693 2024-11-12T10:55:57,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/9f99e64261de423fb077abb968fe625c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9f99e64261de423fb077abb968fe625c 2024-11-12T10:55:57,872 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9f99e64261de423fb077abb968fe625c, entries=150, sequenceid=259, filesize=11.9 K 2024-11-12T10:55:57,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/74289cb4f9074f7db2e87b397dc78248 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/74289cb4f9074f7db2e87b397dc78248 2024-11-12T10:55:57,876 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/74289cb4f9074f7db2e87b397dc78248, entries=150, sequenceid=259, filesize=11.9 K 2024-11-12T10:55:57,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409017867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:57,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/85f3f60804d14eedb1c5ad03a7fb6693 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/85f3f60804d14eedb1c5ad03a7fb6693 2024-11-12T10:55:57,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:57,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:57,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:57,880 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/85f3f60804d14eedb1c5ad03a7fb6693, entries=150, sequenceid=259, filesize=11.9 K 2024-11-12T10:55:57,881 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5b5c37b8175d8b0adea03f092f215997 in 458ms, sequenceid=259, compaction requested=false 2024-11-12T10:55:57,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:57,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:57,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-12T10:55:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-12T10:55:57,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-12T10:55:57,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 612 msec 2024-11-12T10:55:57,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 626 msec 2024-11-12T10:55:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T10:55:57,887 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:57,887 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-12T10:55:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T10:55:57,889 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:57,889 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:57,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:57,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T10:55:58,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-12T10:55:58,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:58,040 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T10:55:58,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:58,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:58,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:58,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:58,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:58,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:58,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/48df28e436a24a4ca38c10ce01d76df2 is 50, key is test_row_0/A:col10/1731408957533/Put/seqid=0 2024-11-12T10:55:58,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742308_1484 (size=12301) 2024-11-12T10:55:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:58,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T10:55:58,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409018268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:58,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:58,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:58,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:58,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409018377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:58,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:58,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:58,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:58,448 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/48df28e436a24a4ca38c10ce01d76df2 2024-11-12T10:55:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/9d1f40c82edf43a89e2ef0032fd96662 is 50, key is test_row_0/B:col10/1731408957533/Put/seqid=0 2024-11-12T10:55:58,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742309_1485 (size=12301) 2024-11-12T10:55:58,467 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/9d1f40c82edf43a89e2ef0032fd96662 2024-11-12T10:55:58,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/f27ce78b86fd4723ae002d4655b644e7 is 50, key is test_row_0/C:col10/1731408957533/Put/seqid=0 2024-11-12T10:55:58,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742310_1486 (size=12301) 2024-11-12T10:55:58,483 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/f27ce78b86fd4723ae002d4655b644e7 2024-11-12T10:55:58,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/48df28e436a24a4ca38c10ce01d76df2 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/48df28e436a24a4ca38c10ce01d76df2 2024-11-12T10:55:58,494 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/48df28e436a24a4ca38c10ce01d76df2, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T10:55:58,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/9d1f40c82edf43a89e2ef0032fd96662 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9d1f40c82edf43a89e2ef0032fd96662 2024-11-12T10:55:58,497 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9d1f40c82edf43a89e2ef0032fd96662, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T10:55:58,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/f27ce78b86fd4723ae002d4655b644e7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/f27ce78b86fd4723ae002d4655b644e7 2024-11-12T10:55:58,501 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/f27ce78b86fd4723ae002d4655b644e7, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T10:55:58,502 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5b5c37b8175d8b0adea03f092f215997 in 461ms, sequenceid=275, compaction requested=true 2024-11-12T10:55:58,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:58,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:58,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-12T10:55:58,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-12T10:55:58,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-12T10:55:58,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 613 msec 2024-11-12T10:55:58,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 617 msec 2024-11-12T10:55:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T10:55:58,517 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:55:58,517 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:55:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-12T10:55:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-12T10:55:58,518 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:55:58,519 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:55:58,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:55:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:58,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T10:55:58,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:58,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:58,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:58,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:58,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:58,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:58,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/10f191449a3442749b926ab4277675b3 is 50, key is test_row_0/A:col10/1731408958590/Put/seqid=0 2024-11-12T10:55:58,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742311_1487 (size=14741) 2024-11-12T10:55:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-12T10:55:58,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-12T10:55:58,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:58,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:58,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:58,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:58,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:58,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:58,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409018669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:58,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:58,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:58,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:58,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409018787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:58,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:58,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:58,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:58,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-12T10:55:58,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:58,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:58,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:58,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:58,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-12T10:55:58,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-12T10:55:58,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:58,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:58,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:58,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:58,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:55:59,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/10f191449a3442749b926ab4277675b3 2024-11-12T10:55:59,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/93135b700d1a40e58cac1b27af364566 is 50, key is test_row_0/B:col10/1731408958590/Put/seqid=0 2024-11-12T10:55:59,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409019007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:59,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:59,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742312_1488 (size=12301) 2024-11-12T10:55:59,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/93135b700d1a40e58cac1b27af364566 2024-11-12T10:55:59,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/84ee6bd1c6fb4b76ab6afb5e5a36b020 is 50, key is test_row_0/C:col10/1731408958590/Put/seqid=0 2024-11-12T10:55:59,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742313_1489 (size=12301) 2024-11-12T10:55:59,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/84ee6bd1c6fb4b76ab6afb5e5a36b020 2024-11-12T10:55:59,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/10f191449a3442749b926ab4277675b3 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/10f191449a3442749b926ab4277675b3 2024-11-12T10:55:59,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/10f191449a3442749b926ab4277675b3, entries=200, sequenceid=292, filesize=14.4 K 2024-11-12T10:55:59,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/93135b700d1a40e58cac1b27af364566 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/93135b700d1a40e58cac1b27af364566 2024-11-12T10:55:59,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/93135b700d1a40e58cac1b27af364566, entries=150, sequenceid=292, filesize=12.0 K 2024-11-12T10:55:59,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/84ee6bd1c6fb4b76ab6afb5e5a36b020 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/84ee6bd1c6fb4b76ab6afb5e5a36b020 2024-11-12T10:55:59,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/84ee6bd1c6fb4b76ab6afb5e5a36b020, entries=150, sequenceid=292, filesize=12.0 K 2024-11-12T10:55:59,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 5b5c37b8175d8b0adea03f092f215997 in 457ms, sequenceid=292, compaction requested=true 2024-11-12T10:55:59,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:59,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:55:59,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:59,050 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:55:59,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:55:59,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:59,050 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:55:59,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:55:59,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:59,050 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51958 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:55:59,050 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:55:59,050 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/A is initiating minor compaction (all files) 2024-11-12T10:55:59,050 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/B is initiating minor compaction (all files) 2024-11-12T10:55:59,050 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/A in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:59,050 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/B in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:59,051 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/0a55f49c147a441b88d2dff80a82b172, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9f99e64261de423fb077abb968fe625c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/48df28e436a24a4ca38c10ce01d76df2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/10f191449a3442749b926ab4277675b3] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=50.7 K 2024-11-12T10:55:59,051 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/ae6ea0ec8b9a47709031dab65f41b477, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/74289cb4f9074f7db2e87b397dc78248, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9d1f40c82edf43a89e2ef0032fd96662, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/93135b700d1a40e58cac1b27af364566] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=48.4 K 2024-11-12T10:55:59,051 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a55f49c147a441b88d2dff80a82b172, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731408955444 2024-11-12T10:55:59,051 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting ae6ea0ec8b9a47709031dab65f41b477, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731408955444 2024-11-12T10:55:59,051 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f99e64261de423fb077abb968fe625c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731408955575 2024-11-12T10:55:59,051 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 74289cb4f9074f7db2e87b397dc78248, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731408955575 2024-11-12T10:55:59,053 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48df28e436a24a4ca38c10ce01d76df2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731408957530 2024-11-12T10:55:59,053 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d1f40c82edf43a89e2ef0032fd96662, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731408957530 2024-11-12T10:55:59,053 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 93135b700d1a40e58cac1b27af364566, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731408958231 2024-11-12T10:55:59,053 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10f191449a3442749b926ab4277675b3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731408958231 2024-11-12T10:55:59,060 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#B#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:59,060 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#A#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:59,060 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/1f17c09fb2af431a9f11c5a046c3864f is 50, key is test_row_0/B:col10/1731408958590/Put/seqid=0 2024-11-12T10:55:59,060 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/598ece2d159c46bfb884b70dfbc84786 is 50, key is test_row_0/A:col10/1731408958590/Put/seqid=0 2024-11-12T10:55:59,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742315_1491 (size=13051) 2024-11-12T10:55:59,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742314_1490 (size=13051) 2024-11-12T10:55:59,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-12T10:55:59,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:59,128 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T10:55:59,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:55:59,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:59,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:55:59,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:59,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:55:59,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:55:59,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/be057b562cd64be5a6d58c510f3388d5 is 50, key is test_row_0/A:col10/1731408958655/Put/seqid=0 2024-11-12T10:55:59,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742316_1492 (size=12301) 2024-11-12T10:55:59,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-12T10:55:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:55:59,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:55:59,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409019431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:59,471 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/1f17c09fb2af431a9f11c5a046c3864f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/1f17c09fb2af431a9f11c5a046c3864f 2024-11-12T10:55:59,474 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/598ece2d159c46bfb884b70dfbc84786 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/598ece2d159c46bfb884b70dfbc84786 2024-11-12T10:55:59,475 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/B of 5b5c37b8175d8b0adea03f092f215997 into 1f17c09fb2af431a9f11c5a046c3864f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:59,475 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:59,475 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/B, priority=12, startTime=1731408959050; duration=0sec 2024-11-12T10:55:59,475 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:55:59,475 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:B 2024-11-12T10:55:59,475 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:55:59,478 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/A of 5b5c37b8175d8b0adea03f092f215997 into 598ece2d159c46bfb884b70dfbc84786(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:59,478 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:59,478 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/A, priority=12, startTime=1731408959049; duration=0sec 2024-11-12T10:55:59,478 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:59,478 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:A 2024-11-12T10:55:59,478 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:55:59,478 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/C is initiating minor compaction (all files) 2024-11-12T10:55:59,478 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/C in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:55:59,479 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/f13d84a32f164effb9f1636e0da6fba2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/85f3f60804d14eedb1c5ad03a7fb6693, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/f27ce78b86fd4723ae002d4655b644e7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/84ee6bd1c6fb4b76ab6afb5e5a36b020] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=48.4 K 2024-11-12T10:55:59,479 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting f13d84a32f164effb9f1636e0da6fba2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731408955444 2024-11-12T10:55:59,479 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 85f3f60804d14eedb1c5ad03a7fb6693, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731408955575 2024-11-12T10:55:59,479 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting f27ce78b86fd4723ae002d4655b644e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731408957530 2024-11-12T10:55:59,480 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 84ee6bd1c6fb4b76ab6afb5e5a36b020, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731408958231 2024-11-12T10:55:59,488 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:55:59,488 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/af17664973914e7d91ec2150efc34d55 is 50, key is test_row_0/C:col10/1731408958590/Put/seqid=0 2024-11-12T10:55:59,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742317_1493 (size=13051) 2024-11-12T10:55:59,542 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/be057b562cd64be5a6d58c510f3388d5 2024-11-12T10:55:59,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/4094c63cbae548abb8236010d0da1bec is 50, key is test_row_0/B:col10/1731408958655/Put/seqid=0 2024-11-12T10:55:59,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409019546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:59,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742318_1494 (size=12301) 2024-11-12T10:55:59,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:59,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-12T10:55:59,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409019757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:59,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:59,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1731409019777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:59,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55054 deadline: 1731409019777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:59,781 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in 5b5c37b8175d8b0adea03f092f215997 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:59,781 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 5b5c37b8175d8b0adea03f092f215997 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:59,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55096 deadline: 1731409019797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:59,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:55:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55124 deadline: 1731409019797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:55:59,801 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in 5b5c37b8175d8b0adea03f092f215997 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:59,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,801 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in 5b5c37b8175d8b0adea03f092f215997 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:55:59,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:59,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:55:59,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:55:59,901 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/af17664973914e7d91ec2150efc34d55 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/af17664973914e7d91ec2150efc34d55 2024-11-12T10:55:59,907 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/C of 5b5c37b8175d8b0adea03f092f215997 into af17664973914e7d91ec2150efc34d55(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:55:59,907 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:55:59,907 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/C, priority=12, startTime=1731408959050; duration=0sec 2024-11-12T10:55:59,907 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:55:59,907 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:C 2024-11-12T10:55:59,951 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/4094c63cbae548abb8236010d0da1bec 2024-11-12T10:55:59,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/95c50e7d54b44b6ead133ee34e3af8b4 is 50, key is test_row_0/C:col10/1731408958655/Put/seqid=0 2024-11-12T10:55:59,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742319_1495 (size=12301) 2024-11-12T10:56:00,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409020077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:56:00,360 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/95c50e7d54b44b6ead133ee34e3af8b4 2024-11-12T10:56:00,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/be057b562cd64be5a6d58c510f3388d5 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/be057b562cd64be5a6d58c510f3388d5 2024-11-12T10:56:00,365 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/be057b562cd64be5a6d58c510f3388d5, entries=150, sequenceid=307, filesize=12.0 K 2024-11-12T10:56:00,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/4094c63cbae548abb8236010d0da1bec as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/4094c63cbae548abb8236010d0da1bec 2024-11-12T10:56:00,369 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/4094c63cbae548abb8236010d0da1bec, entries=150, sequenceid=307, filesize=12.0 K 2024-11-12T10:56:00,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/95c50e7d54b44b6ead133ee34e3af8b4 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/95c50e7d54b44b6ead133ee34e3af8b4 2024-11-12T10:56:00,372 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/95c50e7d54b44b6ead133ee34e3af8b4, entries=150, sequenceid=307, filesize=12.0 K 2024-11-12T10:56:00,373 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 5b5c37b8175d8b0adea03f092f215997 in 1245ms, sequenceid=307, compaction requested=false 2024-11-12T10:56:00,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:56:00,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:00,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-12T10:56:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-12T10:56:00,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-12T10:56:00,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8570 sec 2024-11-12T10:56:00,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.8630 sec 2024-11-12T10:56:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:56:00,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T10:56:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:56:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:56:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:56:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:00,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/f93a48c29a2d4d068cf92bccef4e1c89 is 50, key is test_row_0/A:col10/1731408960599/Put/seqid=0 2024-11-12T10:56:00,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742320_1496 (size=14741) 2024-11-12T10:56:00,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/f93a48c29a2d4d068cf92bccef4e1c89 2024-11-12T10:56:00,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/36f14b84453f4a72b0e3549164477b5c is 50, key is test_row_0/B:col10/1731408960599/Put/seqid=0 2024-11-12T10:56:00,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742321_1497 (size=12301) 2024-11-12T10:56:00,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409020666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-12T10:56:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:56:00,668 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:56:00,668 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:56:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-12T10:56:00,669 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:56:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-12T10:56:00,670 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:56:00,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:56:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-12T10:56:00,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409020777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:56:00,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-12T10:56:00,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:00,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:56:00,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:00,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:00,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:00,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-12T10:56:00,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:00,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:56:00,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:00,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:00,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-12T10:56:01,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409020997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:56:01,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/36f14b84453f4a72b0e3549164477b5c 2024-11-12T10:56:01,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/09dc6f14e93c4b2fbc9020880bf41a9a is 50, key is test_row_0/C:col10/1731408960599/Put/seqid=0 2024-11-12T10:56:01,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742322_1498 (size=12301) 2024-11-12T10:56:01,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-12T10:56:01,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:01,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:56:01,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:01,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:01,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:01,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-12T10:56:01,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:01,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:56:01,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:01,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:01,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-12T10:56:01,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409021307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:56:01,378 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T10:56:01,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-12T10:56:01,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:01,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:56:01,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:01,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:01,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:01,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/09dc6f14e93c4b2fbc9020880bf41a9a 2024-11-12T10:56:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/f93a48c29a2d4d068cf92bccef4e1c89 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/f93a48c29a2d4d068cf92bccef4e1c89 2024-11-12T10:56:01,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/f93a48c29a2d4d068cf92bccef4e1c89, entries=200, sequenceid=328, filesize=14.4 K 2024-11-12T10:56:01,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/36f14b84453f4a72b0e3549164477b5c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/36f14b84453f4a72b0e3549164477b5c 2024-11-12T10:56:01,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/36f14b84453f4a72b0e3549164477b5c, entries=150, sequenceid=328, filesize=12.0 K 2024-11-12T10:56:01,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/09dc6f14e93c4b2fbc9020880bf41a9a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/09dc6f14e93c4b2fbc9020880bf41a9a 2024-11-12T10:56:01,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/09dc6f14e93c4b2fbc9020880bf41a9a, entries=150, sequenceid=328, filesize=12.0 K 2024-11-12T10:56:01,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 5b5c37b8175d8b0adea03f092f215997 in 851ms, sequenceid=328, compaction requested=true 2024-11-12T10:56:01,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:56:01,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:56:01,451 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:01,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:01,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:56:01,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:01,451 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:01,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:56:01,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:01,452 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:01,452 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:01,452 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/B is initiating minor compaction (all files) 2024-11-12T10:56:01,452 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/A is initiating minor compaction (all files) 2024-11-12T10:56:01,452 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/B in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:01,452 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/A in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:01,452 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/1f17c09fb2af431a9f11c5a046c3864f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/4094c63cbae548abb8236010d0da1bec, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/36f14b84453f4a72b0e3549164477b5c] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=36.8 K 2024-11-12T10:56:01,452 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/598ece2d159c46bfb884b70dfbc84786, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/be057b562cd64be5a6d58c510f3388d5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/f93a48c29a2d4d068cf92bccef4e1c89] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=39.2 K 2024-11-12T10:56:01,452 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f17c09fb2af431a9f11c5a046c3864f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731408958231 2024-11-12T10:56:01,452 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 598ece2d159c46bfb884b70dfbc84786, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731408958231 2024-11-12T10:56:01,453 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 4094c63cbae548abb8236010d0da1bec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731408958649 2024-11-12T10:56:01,453 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting be057b562cd64be5a6d58c510f3388d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731408958649 2024-11-12T10:56:01,453 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 36f14b84453f4a72b0e3549164477b5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731408959374 2024-11-12T10:56:01,453 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting f93a48c29a2d4d068cf92bccef4e1c89, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731408959374 2024-11-12T10:56:01,461 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#A#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:01,461 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#B#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:01,461 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/c15f452d8a704962bf49472dec426aaa is 50, key is test_row_0/B:col10/1731408960599/Put/seqid=0 2024-11-12T10:56:01,461 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/18905a7dbc6741e08f53e4c3fbb78774 is 50, key is test_row_0/A:col10/1731408960599/Put/seqid=0 2024-11-12T10:56:01,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742323_1499 (size=13153) 2024-11-12T10:56:01,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742324_1500 (size=13153) 2024-11-12T10:56:01,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-12T10:56:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:01,585 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T10:56:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:56:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:56:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:56:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:01,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/9b1b810151c043b9a7c742206958635c is 50, key is test_row_0/A:col10/1731408960665/Put/seqid=0 2024-11-12T10:56:01,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742325_1501 (size=12301) 2024-11-12T10:56:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-12T10:56:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:56:01,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. as already flushing 2024-11-12T10:56:01,875 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/c15f452d8a704962bf49472dec426aaa as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c15f452d8a704962bf49472dec426aaa 2024-11-12T10:56:01,875 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/18905a7dbc6741e08f53e4c3fbb78774 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/18905a7dbc6741e08f53e4c3fbb78774 2024-11-12T10:56:01,878 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/B of 5b5c37b8175d8b0adea03f092f215997 into c15f452d8a704962bf49472dec426aaa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:01,878 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:56:01,878 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/B, priority=13, startTime=1731408961451; duration=0sec 2024-11-12T10:56:01,878 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:01,878 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:B 2024-11-12T10:56:01,878 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:01,879 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/A of 5b5c37b8175d8b0adea03f092f215997 into 18905a7dbc6741e08f53e4c3fbb78774(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:01,879 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:56:01,879 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/A, priority=13, startTime=1731408961451; duration=0sec 2024-11-12T10:56:01,879 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:01,879 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:A 2024-11-12T10:56:01,879 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:01,879 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/C is initiating minor compaction (all files) 2024-11-12T10:56:01,879 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/C in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:01,880 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/af17664973914e7d91ec2150efc34d55, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/95c50e7d54b44b6ead133ee34e3af8b4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/09dc6f14e93c4b2fbc9020880bf41a9a] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=36.8 K 2024-11-12T10:56:01,880 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting af17664973914e7d91ec2150efc34d55, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731408958231 2024-11-12T10:56:01,880 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 95c50e7d54b44b6ead133ee34e3af8b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731408958649 2024-11-12T10:56:01,880 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 09dc6f14e93c4b2fbc9020880bf41a9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731408959374 2024-11-12T10:56:01,885 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#C#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:01,885 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/10b4c38a0aca406faaab4c2bec950ee9 is 50, key is test_row_0/C:col10/1731408960599/Put/seqid=0 2024-11-12T10:56:01,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742326_1502 (size=13153) 2024-11-12T10:56:01,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409021923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:56:01,992 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/9b1b810151c043b9a7c742206958635c 2024-11-12T10:56:01,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/55e4ed5d14ed489ab4a72d8aeb59004a is 50, key is test_row_0/B:col10/1731408960665/Put/seqid=0 2024-11-12T10:56:02,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742327_1503 (size=12301) 2024-11-12T10:56:02,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409022037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:56:02,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409022246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:56:02,292 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/10b4c38a0aca406faaab4c2bec950ee9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/10b4c38a0aca406faaab4c2bec950ee9 2024-11-12T10:56:02,295 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/C of 5b5c37b8175d8b0adea03f092f215997 into 10b4c38a0aca406faaab4c2bec950ee9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:02,295 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:56:02,295 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/C, priority=13, startTime=1731408961451; duration=0sec 2024-11-12T10:56:02,295 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:02,295 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:C 2024-11-12T10:56:02,345 INFO [Thread-1926 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1926. 2024-11-12T10:56:02,345 DEBUG [Thread-1926 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:02,345 DEBUG [Thread-1926 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:02,345 DEBUG [Thread-1926 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:02,345 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:02,348 INFO [Thread-1928 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1928. 2024-11-12T10:56:02,348 DEBUG [Thread-1928 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:02,348 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:02,348 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:02,348 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:02,351 INFO [Thread-1930 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1930. 2024-11-12T10:56:02,351 DEBUG [Thread-1930 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:02,351 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:02,351 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:02,351 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:02,353 INFO [Thread-1932 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1932. 2024-11-12T10:56:02,353 DEBUG [Thread-1932 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:02,353 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:02,353 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:02,353 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:02,356 INFO [Thread-1934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1934. 2024-11-12T10:56:02,356 DEBUG [Thread-1934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:02,356 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:02,356 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:02,356 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:02,401 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/55e4ed5d14ed489ab4a72d8aeb59004a 2024-11-12T10:56:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/26449dccd9a84591913eda90569aef7b is 50, key is test_row_0/C:col10/1731408960665/Put/seqid=0 2024-11-12T10:56:02,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742328_1504 (size=12301) 2024-11-12T10:56:02,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1731409022567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 , the old value is region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., hostname=aed202db7571,43143,1731408812834, seqNum=2 because the exception is null or not the one we care about 2024-11-12T10:56:02,810 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/26449dccd9a84591913eda90569aef7b 2024-11-12T10:56:02,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/9b1b810151c043b9a7c742206958635c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9b1b810151c043b9a7c742206958635c 2024-11-12T10:56:02,826 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9b1b810151c043b9a7c742206958635c, entries=150, sequenceid=342, filesize=12.0 K 2024-11-12T10:56:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-12T10:56:02,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/55e4ed5d14ed489ab4a72d8aeb59004a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/55e4ed5d14ed489ab4a72d8aeb59004a 2024-11-12T10:56:02,832 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/55e4ed5d14ed489ab4a72d8aeb59004a, entries=150, sequenceid=342, filesize=12.0 K 2024-11-12T10:56:02,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/26449dccd9a84591913eda90569aef7b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/26449dccd9a84591913eda90569aef7b 2024-11-12T10:56:02,837 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/26449dccd9a84591913eda90569aef7b, entries=150, sequenceid=342, filesize=12.0 K 2024-11-12T10:56:02,838 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5b5c37b8175d8b0adea03f092f215997 in 1253ms, sequenceid=342, compaction requested=false 2024-11-12T10:56:02,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:56:02,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:02,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-12T10:56:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-12T10:56:02,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-12T10:56:02,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1700 sec 2024-11-12T10:56:02,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.1740 sec 2024-11-12T10:56:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:56:03,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T10:56:03,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:56:03,080 INFO [Thread-1923 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1923. 2024-11-12T10:56:03,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:03,080 DEBUG [Thread-1923 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:03,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:56:03,080 DEBUG [Thread-1923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:03,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:03,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:56:03,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:03,081 DEBUG [Thread-1923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:03,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:03,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/e2195c5f7c754f24b04b98be3bc871ff is 50, key is test_row_0/A:col10/1731408961861/Put/seqid=0 2024-11-12T10:56:03,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742329_1505 (size=12301) 2024-11-12T10:56:03,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/e2195c5f7c754f24b04b98be3bc871ff 2024-11-12T10:56:03,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/dc7ed1a0f25e4fc1bf16aed288021e12 is 50, key is test_row_0/B:col10/1731408961861/Put/seqid=0 2024-11-12T10:56:03,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742330_1506 (size=12301) 2024-11-12T10:56:03,801 INFO [Thread-1919 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1919. 2024-11-12T10:56:03,801 DEBUG [Thread-1919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:03,801 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:03,801 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:03,801 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:03,811 INFO [Thread-1917 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1917. 2024-11-12T10:56:03,811 DEBUG [Thread-1917 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:03,811 DEBUG [Thread-1917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:03,811 DEBUG [Thread-1917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:03,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:03,831 INFO [Thread-1915 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1915. 2024-11-12T10:56:03,831 DEBUG [Thread-1915 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:03,831 DEBUG [Thread-1915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:03,831 DEBUG [Thread-1915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:03,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:03,848 INFO [Thread-1921 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1921. 2024-11-12T10:56:03,848 DEBUG [Thread-1921 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:03,848 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:03,848 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:03,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:03,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/dc7ed1a0f25e4fc1bf16aed288021e12 2024-11-12T10:56:03,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/1e07cfffcc2f4180b89255e3ca64095b is 50, key is test_row_0/C:col10/1731408961861/Put/seqid=0 2024-11-12T10:56:03,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742331_1507 (size=12301) 2024-11-12T10:56:04,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/1e07cfffcc2f4180b89255e3ca64095b 2024-11-12T10:56:04,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/e2195c5f7c754f24b04b98be3bc871ff as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/e2195c5f7c754f24b04b98be3bc871ff 2024-11-12T10:56:04,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/e2195c5f7c754f24b04b98be3bc871ff, entries=150, sequenceid=364, filesize=12.0 K 2024-11-12T10:56:04,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/dc7ed1a0f25e4fc1bf16aed288021e12 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/dc7ed1a0f25e4fc1bf16aed288021e12 2024-11-12T10:56:04,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/dc7ed1a0f25e4fc1bf16aed288021e12, entries=150, sequenceid=364, filesize=12.0 K 2024-11-12T10:56:04,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/1e07cfffcc2f4180b89255e3ca64095b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/1e07cfffcc2f4180b89255e3ca64095b 2024-11-12T10:56:04,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/1e07cfffcc2f4180b89255e3ca64095b, entries=150, sequenceid=364, filesize=12.0 K 2024-11-12T10:56:04,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=26.84 KB/27480 for 5b5c37b8175d8b0adea03f092f215997 in 1264ms, sequenceid=364, compaction requested=true 2024-11-12T10:56:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:56:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:56:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:04,345 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:56:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b5c37b8175d8b0adea03f092f215997:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:56:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:04,345 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:04,346 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:04,347 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:04,347 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/A is initiating minor compaction (all files) 2024-11-12T10:56:04,347 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/B is initiating minor compaction (all files) 2024-11-12T10:56:04,347 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/A in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:04,347 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/B in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:04,347 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/18905a7dbc6741e08f53e4c3fbb78774, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9b1b810151c043b9a7c742206958635c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/e2195c5f7c754f24b04b98be3bc871ff] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=36.9 K 2024-11-12T10:56:04,347 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c15f452d8a704962bf49472dec426aaa, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/55e4ed5d14ed489ab4a72d8aeb59004a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/dc7ed1a0f25e4fc1bf16aed288021e12] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=36.9 K 2024-11-12T10:56:04,347 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18905a7dbc6741e08f53e4c3fbb78774, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731408959374 2024-11-12T10:56:04,347 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting c15f452d8a704962bf49472dec426aaa, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731408959374 2024-11-12T10:56:04,348 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b1b810151c043b9a7c742206958635c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731408960646 2024-11-12T10:56:04,348 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 55e4ed5d14ed489ab4a72d8aeb59004a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731408960646 2024-11-12T10:56:04,349 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2195c5f7c754f24b04b98be3bc871ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731408961861 2024-11-12T10:56:04,349 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting dc7ed1a0f25e4fc1bf16aed288021e12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731408961861 2024-11-12T10:56:04,357 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#A#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:04,357 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:04,357 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/bbe57ea12b0e455abdb0431d79dfb44d is 50, key is test_row_0/A:col10/1731408961861/Put/seqid=0 2024-11-12T10:56:04,357 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/a7f679b37a5a422b972537be1febf579 is 50, key is test_row_0/B:col10/1731408961861/Put/seqid=0 2024-11-12T10:56:04,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742333_1509 (size=13255) 2024-11-12T10:56:04,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742332_1508 (size=13255) 2024-11-12T10:56:04,770 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/a7f679b37a5a422b972537be1febf579 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/a7f679b37a5a422b972537be1febf579 2024-11-12T10:56:04,770 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/bbe57ea12b0e455abdb0431d79dfb44d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/bbe57ea12b0e455abdb0431d79dfb44d 2024-11-12T10:56:04,776 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/A of 5b5c37b8175d8b0adea03f092f215997 into bbe57ea12b0e455abdb0431d79dfb44d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:04,776 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/B of 5b5c37b8175d8b0adea03f092f215997 into a7f679b37a5a422b972537be1febf579(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:04,776 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:56:04,776 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:56:04,776 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/A, priority=13, startTime=1731408964345; duration=0sec 2024-11-12T10:56:04,776 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/B, priority=13, startTime=1731408964345; duration=0sec 2024-11-12T10:56:04,776 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:04,777 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:B 2024-11-12T10:56:04,777 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:04,777 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:A 2024-11-12T10:56:04,777 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:04,778 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:04,778 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): 5b5c37b8175d8b0adea03f092f215997/C is initiating minor compaction (all files) 2024-11-12T10:56:04,778 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b5c37b8175d8b0adea03f092f215997/C in TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:04,778 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/10b4c38a0aca406faaab4c2bec950ee9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/26449dccd9a84591913eda90569aef7b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/1e07cfffcc2f4180b89255e3ca64095b] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp, totalSize=36.9 K 2024-11-12T10:56:04,779 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 10b4c38a0aca406faaab4c2bec950ee9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731408959374 2024-11-12T10:56:04,779 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 26449dccd9a84591913eda90569aef7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731408960646 2024-11-12T10:56:04,779 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e07cfffcc2f4180b89255e3ca64095b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731408961861 2024-11-12T10:56:04,788 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b5c37b8175d8b0adea03f092f215997#C#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:04,788 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/062f90d039324bd69a6e027b674fb513 is 50, key is test_row_0/C:col10/1731408961861/Put/seqid=0 2024-11-12T10:56:04,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742334_1510 (size=13255) 2024-11-12T10:56:04,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-12T10:56:04,847 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1291 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3873 rows 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1286 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3855 rows 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1289 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3867 rows 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1294 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3879 rows 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1291 2024-11-12T10:56:04,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3873 rows 2024-11-12T10:56:04,848 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T10:56:04,848 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T10:56:04,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:04,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:04,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:04,849 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T10:56:04,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:04,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-12T10:56:04,851 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408964851"}]},"ts":"1731408964851"} 2024-11-12T10:56:04,853 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T10:56:04,853 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T10:56:04,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T10:56:04,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b5c37b8175d8b0adea03f092f215997, UNASSIGN}] 2024-11-12T10:56:04,855 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b5c37b8175d8b0adea03f092f215997, UNASSIGN 2024-11-12T10:56:04,855 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=5b5c37b8175d8b0adea03f092f215997, regionState=CLOSING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:56:04,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b5c37b8175d8b0adea03f092f215997, UNASSIGN because future has completed 2024-11-12T10:56:04,857 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T10:56:04,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:56:04,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-12T10:56:05,009 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:56:05,009 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T10:56:05,009 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 5b5c37b8175d8b0adea03f092f215997, disabling compactions & flushes 2024-11-12T10:56:05,009 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-12T10:56:05,201 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/062f90d039324bd69a6e027b674fb513 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/062f90d039324bd69a6e027b674fb513 2024-11-12T10:56:05,207 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b5c37b8175d8b0adea03f092f215997/C of 5b5c37b8175d8b0adea03f092f215997 into 062f90d039324bd69a6e027b674fb513(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:05,207 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b5c37b8175d8b0adea03f092f215997: 2024-11-12T10:56:05,207 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:05,207 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., storeName=5b5c37b8175d8b0adea03f092f215997/C, priority=13, startTime=1731408964345; duration=0sec 2024-11-12T10:56:05,207 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:05,207 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. after waiting 0 ms 2024-11-12T10:56:05,207 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:05,207 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:05,207 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b5c37b8175d8b0adea03f092f215997:C 2024-11-12T10:56:05,207 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing 5b5c37b8175d8b0adea03f092f215997 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T10:56:05,208 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=A 2024-11-12T10:56:05,208 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:05,208 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=B 2024-11-12T10:56:05,208 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:05,208 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b5c37b8175d8b0adea03f092f215997, store=C 2024-11-12T10:56:05,208 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:05,212 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/da63b95b898c4dc0a56c5c55d5ddbca7 is 50, key is test_row_1/A:col10/1731408963828/Put/seqid=0 2024-11-12T10:56:05,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742335_1511 (size=9857) 2024-11-12T10:56:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-12T10:56:05,616 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/da63b95b898c4dc0a56c5c55d5ddbca7 2024-11-12T10:56:05,621 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/1a7ca16a311145f2834c99adc18691c1 is 50, key is test_row_1/B:col10/1731408963828/Put/seqid=0 2024-11-12T10:56:05,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742336_1512 (size=9857) 2024-11-12T10:56:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-12T10:56:06,026 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/1a7ca16a311145f2834c99adc18691c1 2024-11-12T10:56:06,040 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/c2f5425cec2d43f69cd9a63a8b0ecb92 is 50, key is test_row_1/C:col10/1731408963828/Put/seqid=0 2024-11-12T10:56:06,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742337_1513 (size=9857) 2024-11-12T10:56:06,444 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/c2f5425cec2d43f69cd9a63a8b0ecb92 2024-11-12T10:56:06,450 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/A/da63b95b898c4dc0a56c5c55d5ddbca7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/da63b95b898c4dc0a56c5c55d5ddbca7 2024-11-12T10:56:06,455 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/da63b95b898c4dc0a56c5c55d5ddbca7, entries=100, sequenceid=374, filesize=9.6 K 2024-11-12T10:56:06,456 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/B/1a7ca16a311145f2834c99adc18691c1 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/1a7ca16a311145f2834c99adc18691c1 2024-11-12T10:56:06,458 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/1a7ca16a311145f2834c99adc18691c1, entries=100, sequenceid=374, filesize=9.6 K 2024-11-12T10:56:06,459 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/.tmp/C/c2f5425cec2d43f69cd9a63a8b0ecb92 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c2f5425cec2d43f69cd9a63a8b0ecb92 2024-11-12T10:56:06,461 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c2f5425cec2d43f69cd9a63a8b0ecb92, entries=100, sequenceid=374, filesize=9.6 K 2024-11-12T10:56:06,462 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5b5c37b8175d8b0adea03f092f215997 in 1255ms, sequenceid=374, compaction requested=false 2024-11-12T10:56:06,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/bd9219b3688f4dc88287ec74b938c90e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6c481e680fc94ce5bd96ce81d936077d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6592621145af4087ba24f76cbedcc2a1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/d0e03d3562fd4b3282300fe602bfb444, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/cde43de0b2a1473696f634ed8b73e387, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/ddaa075745824ce4b001ba762e9c81cf, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/132a481bd2b1419f8cc3dbca9661e07b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/4c51d67c31f54e66bf71f96ca2c5b3de, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9825e90f1b3b4bcca47e24ac20a429c4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/71d6d4aff837405fb7fb65dda8e69d57, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/834b7c2ac6ed445c9d785f72bfecb664, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/493dc215fc9f447896b48f3a1cd9d019, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/045472d13dc44c328428c0e1bf40814d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/5586409a4bae42a190d5137e3d16e8b8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/641c76a954694ad0886f5c56d62f1241, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/a6d033d130f542c8b3e355f00fb6ec9b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6ceefd5425fc4d608861b34fb91c3e45, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/13a762f8db96498582b33773e4dc92a9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/0a55f49c147a441b88d2dff80a82b172, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9f99e64261de423fb077abb968fe625c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/48df28e436a24a4ca38c10ce01d76df2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/10f191449a3442749b926ab4277675b3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/598ece2d159c46bfb884b70dfbc84786, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/be057b562cd64be5a6d58c510f3388d5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/f93a48c29a2d4d068cf92bccef4e1c89, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/18905a7dbc6741e08f53e4c3fbb78774, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9b1b810151c043b9a7c742206958635c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/e2195c5f7c754f24b04b98be3bc871ff] to archive 2024-11-12T10:56:06,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:56:06,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/bd9219b3688f4dc88287ec74b938c90e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/bd9219b3688f4dc88287ec74b938c90e 2024-11-12T10:56:06,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6c481e680fc94ce5bd96ce81d936077d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6c481e680fc94ce5bd96ce81d936077d 2024-11-12T10:56:06,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6592621145af4087ba24f76cbedcc2a1 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6592621145af4087ba24f76cbedcc2a1 2024-11-12T10:56:06,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/d0e03d3562fd4b3282300fe602bfb444 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/d0e03d3562fd4b3282300fe602bfb444 2024-11-12T10:56:06,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/cde43de0b2a1473696f634ed8b73e387 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/cde43de0b2a1473696f634ed8b73e387 2024-11-12T10:56:06,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/ddaa075745824ce4b001ba762e9c81cf to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/ddaa075745824ce4b001ba762e9c81cf 2024-11-12T10:56:06,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/132a481bd2b1419f8cc3dbca9661e07b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/132a481bd2b1419f8cc3dbca9661e07b 2024-11-12T10:56:06,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/4c51d67c31f54e66bf71f96ca2c5b3de to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/4c51d67c31f54e66bf71f96ca2c5b3de 2024-11-12T10:56:06,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9825e90f1b3b4bcca47e24ac20a429c4 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9825e90f1b3b4bcca47e24ac20a429c4 2024-11-12T10:56:06,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/71d6d4aff837405fb7fb65dda8e69d57 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/71d6d4aff837405fb7fb65dda8e69d57 2024-11-12T10:56:06,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/834b7c2ac6ed445c9d785f72bfecb664 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/834b7c2ac6ed445c9d785f72bfecb664 2024-11-12T10:56:06,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/493dc215fc9f447896b48f3a1cd9d019 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/493dc215fc9f447896b48f3a1cd9d019 2024-11-12T10:56:06,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/045472d13dc44c328428c0e1bf40814d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/045472d13dc44c328428c0e1bf40814d 2024-11-12T10:56:06,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/5586409a4bae42a190d5137e3d16e8b8 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/5586409a4bae42a190d5137e3d16e8b8 2024-11-12T10:56:06,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/641c76a954694ad0886f5c56d62f1241 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/641c76a954694ad0886f5c56d62f1241 2024-11-12T10:56:06,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/a6d033d130f542c8b3e355f00fb6ec9b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/a6d033d130f542c8b3e355f00fb6ec9b 2024-11-12T10:56:06,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6ceefd5425fc4d608861b34fb91c3e45 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/6ceefd5425fc4d608861b34fb91c3e45 2024-11-12T10:56:06,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/13a762f8db96498582b33773e4dc92a9 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/13a762f8db96498582b33773e4dc92a9 2024-11-12T10:56:06,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/0a55f49c147a441b88d2dff80a82b172 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/0a55f49c147a441b88d2dff80a82b172 2024-11-12T10:56:06,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9f99e64261de423fb077abb968fe625c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9f99e64261de423fb077abb968fe625c 2024-11-12T10:56:06,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/48df28e436a24a4ca38c10ce01d76df2 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/48df28e436a24a4ca38c10ce01d76df2 2024-11-12T10:56:06,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/10f191449a3442749b926ab4277675b3 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/10f191449a3442749b926ab4277675b3 2024-11-12T10:56:06,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/598ece2d159c46bfb884b70dfbc84786 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/598ece2d159c46bfb884b70dfbc84786 2024-11-12T10:56:06,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/be057b562cd64be5a6d58c510f3388d5 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/be057b562cd64be5a6d58c510f3388d5 2024-11-12T10:56:06,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/f93a48c29a2d4d068cf92bccef4e1c89 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/f93a48c29a2d4d068cf92bccef4e1c89 2024-11-12T10:56:06,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/18905a7dbc6741e08f53e4c3fbb78774 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/18905a7dbc6741e08f53e4c3fbb78774 2024-11-12T10:56:06,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9b1b810151c043b9a7c742206958635c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/9b1b810151c043b9a7c742206958635c 2024-11-12T10:56:06,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/e2195c5f7c754f24b04b98be3bc871ff to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/e2195c5f7c754f24b04b98be3bc871ff 2024-11-12T10:56:06,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/15c0a655fe014ad68b43a2147187aee2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/e7ce41c1380e4a868c2d207e4a5a3d44, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/523b02e7e8b64901949d2888aeb836e0, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/8de64ab9053a472fbdbe7e5fdd58f9b4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c8f15cac3cb04f9f822a294d47e51d6a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/fb1249c3fb0e4f08b48b3007d001bf28, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/007dab79063c465cb424ae6bdba1ccae, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/55a7262951344379b7baec9626766e94, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/5e03d7d6779b4b798c8c3ad75bbc6962, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c0307e0817994bba8abbf6b5db127eaa, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/64f1b6d3e8fa4f5a805196d2457ba499, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/2c228ecf71ac41f4a1f6511b6f240455, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/5a71f28f82d64037a3328bcdb5427919, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9f438e19392c4e66b0145362f729e456, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/189441efc45c44c3ad520499f4ef50d8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/613da75614d347639f4f40f7dbdde295, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9b74276cfd61441aab59c32793c35cd9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/ae6ea0ec8b9a47709031dab65f41b477, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/10f8de0a262b4268a13c5172dc4419d7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/74289cb4f9074f7db2e87b397dc78248, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9d1f40c82edf43a89e2ef0032fd96662, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/1f17c09fb2af431a9f11c5a046c3864f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/93135b700d1a40e58cac1b27af364566, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/4094c63cbae548abb8236010d0da1bec, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c15f452d8a704962bf49472dec426aaa, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/36f14b84453f4a72b0e3549164477b5c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/55e4ed5d14ed489ab4a72d8aeb59004a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/dc7ed1a0f25e4fc1bf16aed288021e12] to archive 2024-11-12T10:56:06,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:56:06,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/15c0a655fe014ad68b43a2147187aee2 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/15c0a655fe014ad68b43a2147187aee2 2024-11-12T10:56:06,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/e7ce41c1380e4a868c2d207e4a5a3d44 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/e7ce41c1380e4a868c2d207e4a5a3d44 2024-11-12T10:56:06,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/523b02e7e8b64901949d2888aeb836e0 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/523b02e7e8b64901949d2888aeb836e0 2024-11-12T10:56:06,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/8de64ab9053a472fbdbe7e5fdd58f9b4 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/8de64ab9053a472fbdbe7e5fdd58f9b4 2024-11-12T10:56:06,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c8f15cac3cb04f9f822a294d47e51d6a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c8f15cac3cb04f9f822a294d47e51d6a 2024-11-12T10:56:06,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/fb1249c3fb0e4f08b48b3007d001bf28 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/fb1249c3fb0e4f08b48b3007d001bf28 2024-11-12T10:56:06,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/007dab79063c465cb424ae6bdba1ccae to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/007dab79063c465cb424ae6bdba1ccae 2024-11-12T10:56:06,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/55a7262951344379b7baec9626766e94 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/55a7262951344379b7baec9626766e94 2024-11-12T10:56:06,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/5e03d7d6779b4b798c8c3ad75bbc6962 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/5e03d7d6779b4b798c8c3ad75bbc6962 2024-11-12T10:56:06,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c0307e0817994bba8abbf6b5db127eaa to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c0307e0817994bba8abbf6b5db127eaa 2024-11-12T10:56:06,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/64f1b6d3e8fa4f5a805196d2457ba499 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/64f1b6d3e8fa4f5a805196d2457ba499 2024-11-12T10:56:06,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/2c228ecf71ac41f4a1f6511b6f240455 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/2c228ecf71ac41f4a1f6511b6f240455 2024-11-12T10:56:06,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/5a71f28f82d64037a3328bcdb5427919 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/5a71f28f82d64037a3328bcdb5427919 2024-11-12T10:56:06,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9f438e19392c4e66b0145362f729e456 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9f438e19392c4e66b0145362f729e456 2024-11-12T10:56:06,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/189441efc45c44c3ad520499f4ef50d8 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/189441efc45c44c3ad520499f4ef50d8 2024-11-12T10:56:06,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/613da75614d347639f4f40f7dbdde295 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/613da75614d347639f4f40f7dbdde295 2024-11-12T10:56:06,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9b74276cfd61441aab59c32793c35cd9 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9b74276cfd61441aab59c32793c35cd9 2024-11-12T10:56:06,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/ae6ea0ec8b9a47709031dab65f41b477 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/ae6ea0ec8b9a47709031dab65f41b477 2024-11-12T10:56:06,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/10f8de0a262b4268a13c5172dc4419d7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/10f8de0a262b4268a13c5172dc4419d7 2024-11-12T10:56:06,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/74289cb4f9074f7db2e87b397dc78248 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/74289cb4f9074f7db2e87b397dc78248 2024-11-12T10:56:06,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9d1f40c82edf43a89e2ef0032fd96662 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/9d1f40c82edf43a89e2ef0032fd96662 2024-11-12T10:56:06,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/1f17c09fb2af431a9f11c5a046c3864f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/1f17c09fb2af431a9f11c5a046c3864f 2024-11-12T10:56:06,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/93135b700d1a40e58cac1b27af364566 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/93135b700d1a40e58cac1b27af364566 2024-11-12T10:56:06,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/4094c63cbae548abb8236010d0da1bec to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/4094c63cbae548abb8236010d0da1bec 2024-11-12T10:56:06,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c15f452d8a704962bf49472dec426aaa to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/c15f452d8a704962bf49472dec426aaa 2024-11-12T10:56:06,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/36f14b84453f4a72b0e3549164477b5c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/36f14b84453f4a72b0e3549164477b5c 2024-11-12T10:56:06,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/55e4ed5d14ed489ab4a72d8aeb59004a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/55e4ed5d14ed489ab4a72d8aeb59004a 2024-11-12T10:56:06,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/dc7ed1a0f25e4fc1bf16aed288021e12 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/dc7ed1a0f25e4fc1bf16aed288021e12 2024-11-12T10:56:06,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9a0ae2aee19f4634837d7106730380dc, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c2c9e0aa73b84d4a8a7b563eb063046d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/0ed063df3bf5431ca58fe4b5052c518c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/ec49d3c719c045f9bdb6ff75c74bd50b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/e1f1ea4fb937441fa031a748a20eb7cb, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/74fd77ce97db4bbb9235a9a2e0ab203e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/7321772a465c4cc6ae3d23a6d6b5a9ce, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/09d15d4e7cff4263b4fc8eb542f83618, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/d807cd07d90b4554bc1a22e7e7589a16, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9b5bd5f1079541db84bf9caa84f5fb59, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/19808052c8684f67968cbedf9ce573d6, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/682b1a30af1d4ff1a1e20cbe359719ad, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c3e6f73aee90431fb068f14485c35841, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/b578285d246045d3b24998d70709be2e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/b492b6bac20e43e38f027026a83c33d4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/8e250ec01d364c6fa814110c9bebc6b6, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9388dfeaf52141a79a80e9385337fdb8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/f13d84a32f164effb9f1636e0da6fba2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/4101fcb2ba1b4ad4aaf9f08295979e91, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/85f3f60804d14eedb1c5ad03a7fb6693, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/f27ce78b86fd4723ae002d4655b644e7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/af17664973914e7d91ec2150efc34d55, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/84ee6bd1c6fb4b76ab6afb5e5a36b020, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/95c50e7d54b44b6ead133ee34e3af8b4, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/10b4c38a0aca406faaab4c2bec950ee9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/09dc6f14e93c4b2fbc9020880bf41a9a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/26449dccd9a84591913eda90569aef7b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/1e07cfffcc2f4180b89255e3ca64095b] to archive 2024-11-12T10:56:06,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:56:06,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9a0ae2aee19f4634837d7106730380dc to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9a0ae2aee19f4634837d7106730380dc 2024-11-12T10:56:06,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c2c9e0aa73b84d4a8a7b563eb063046d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c2c9e0aa73b84d4a8a7b563eb063046d 2024-11-12T10:56:06,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/0ed063df3bf5431ca58fe4b5052c518c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/0ed063df3bf5431ca58fe4b5052c518c 2024-11-12T10:56:06,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/ec49d3c719c045f9bdb6ff75c74bd50b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/ec49d3c719c045f9bdb6ff75c74bd50b 2024-11-12T10:56:06,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/e1f1ea4fb937441fa031a748a20eb7cb to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/e1f1ea4fb937441fa031a748a20eb7cb 2024-11-12T10:56:06,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/74fd77ce97db4bbb9235a9a2e0ab203e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/74fd77ce97db4bbb9235a9a2e0ab203e 2024-11-12T10:56:06,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/7321772a465c4cc6ae3d23a6d6b5a9ce to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/7321772a465c4cc6ae3d23a6d6b5a9ce 2024-11-12T10:56:06,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/09d15d4e7cff4263b4fc8eb542f83618 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/09d15d4e7cff4263b4fc8eb542f83618 2024-11-12T10:56:06,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/d807cd07d90b4554bc1a22e7e7589a16 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/d807cd07d90b4554bc1a22e7e7589a16 2024-11-12T10:56:06,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9b5bd5f1079541db84bf9caa84f5fb59 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9b5bd5f1079541db84bf9caa84f5fb59 2024-11-12T10:56:06,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/19808052c8684f67968cbedf9ce573d6 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/19808052c8684f67968cbedf9ce573d6 2024-11-12T10:56:06,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/682b1a30af1d4ff1a1e20cbe359719ad to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/682b1a30af1d4ff1a1e20cbe359719ad 2024-11-12T10:56:06,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c3e6f73aee90431fb068f14485c35841 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c3e6f73aee90431fb068f14485c35841 2024-11-12T10:56:06,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/b578285d246045d3b24998d70709be2e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/b578285d246045d3b24998d70709be2e 2024-11-12T10:56:06,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/b492b6bac20e43e38f027026a83c33d4 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/b492b6bac20e43e38f027026a83c33d4 2024-11-12T10:56:06,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/8e250ec01d364c6fa814110c9bebc6b6 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/8e250ec01d364c6fa814110c9bebc6b6 2024-11-12T10:56:06,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9388dfeaf52141a79a80e9385337fdb8 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/9388dfeaf52141a79a80e9385337fdb8 2024-11-12T10:56:06,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/f13d84a32f164effb9f1636e0da6fba2 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/f13d84a32f164effb9f1636e0da6fba2 2024-11-12T10:56:06,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/4101fcb2ba1b4ad4aaf9f08295979e91 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/4101fcb2ba1b4ad4aaf9f08295979e91 2024-11-12T10:56:06,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/85f3f60804d14eedb1c5ad03a7fb6693 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/85f3f60804d14eedb1c5ad03a7fb6693 2024-11-12T10:56:06,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/f27ce78b86fd4723ae002d4655b644e7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/f27ce78b86fd4723ae002d4655b644e7 2024-11-12T10:56:06,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/af17664973914e7d91ec2150efc34d55 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/af17664973914e7d91ec2150efc34d55 2024-11-12T10:56:06,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/84ee6bd1c6fb4b76ab6afb5e5a36b020 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/84ee6bd1c6fb4b76ab6afb5e5a36b020 2024-11-12T10:56:06,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/95c50e7d54b44b6ead133ee34e3af8b4 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/95c50e7d54b44b6ead133ee34e3af8b4 2024-11-12T10:56:06,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/10b4c38a0aca406faaab4c2bec950ee9 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/10b4c38a0aca406faaab4c2bec950ee9 2024-11-12T10:56:06,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/09dc6f14e93c4b2fbc9020880bf41a9a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/09dc6f14e93c4b2fbc9020880bf41a9a 2024-11-12T10:56:06,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/26449dccd9a84591913eda90569aef7b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/26449dccd9a84591913eda90569aef7b 2024-11-12T10:56:06,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/1e07cfffcc2f4180b89255e3ca64095b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/1e07cfffcc2f4180b89255e3ca64095b 2024-11-12T10:56:06,528 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/recovered.edits/377.seqid, newMaxSeqId=377, maxSeqId=1 2024-11-12T10:56:06,528 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. 2024-11-12T10:56:06,528 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 5b5c37b8175d8b0adea03f092f215997: Waiting for close lock at 1731408965009Running coprocessor pre-close hooks at 1731408965009Disabling compacts and flushes for region at 1731408965009Disabling writes for close at 1731408965207 (+198 ms)Obtaining lock to block concurrent updates at 1731408965207Preparing flush snapshotting stores in 5b5c37b8175d8b0adea03f092f215997 at 1731408965207Finished memstore snapshotting TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731408965208 (+1 ms)Flushing stores of TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997. at 1731408965208Flushing 5b5c37b8175d8b0adea03f092f215997/A: creating writer at 1731408965208Flushing 5b5c37b8175d8b0adea03f092f215997/A: appending metadata at 1731408965211 (+3 ms)Flushing 5b5c37b8175d8b0adea03f092f215997/A: closing flushed file at 1731408965211Flushing 5b5c37b8175d8b0adea03f092f215997/B: creating writer at 1731408965619 (+408 ms)Flushing 5b5c37b8175d8b0adea03f092f215997/B: appending metadata at 1731408965621 (+2 ms)Flushing 5b5c37b8175d8b0adea03f092f215997/B: closing flushed file at 1731408965621Flushing 5b5c37b8175d8b0adea03f092f215997/C: creating writer at 1731408966034 (+413 ms)Flushing 5b5c37b8175d8b0adea03f092f215997/C: appending metadata at 1731408966039 (+5 ms)Flushing 5b5c37b8175d8b0adea03f092f215997/C: closing flushed file at 1731408966039Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60111c69: reopening flushed file at 1731408966449 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e4f664d: reopening flushed file at 1731408966455 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@753c888b: reopening flushed file at 1731408966458 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5b5c37b8175d8b0adea03f092f215997 in 1255ms, sequenceid=374, compaction requested=false at 1731408966462 (+4 ms)Writing region close event to WAL at 1731408966525 (+63 ms)Running coprocessor post-close hooks at 1731408966528 (+3 ms)Closed at 1731408966528 2024-11-12T10:56:06,530 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:56:06,530 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=5b5c37b8175d8b0adea03f092f215997, regionState=CLOSED 2024-11-12T10:56:06,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:56:06,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-12T10:56:06,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 5b5c37b8175d8b0adea03f092f215997, server=aed202db7571,43143,1731408812834 in 1.6750 sec 2024-11-12T10:56:06,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-12T10:56:06,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b5c37b8175d8b0adea03f092f215997, UNASSIGN in 1.6790 sec 2024-11-12T10:56:06,536 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-12T10:56:06,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6820 sec 2024-11-12T10:56:06,538 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408966537"}]},"ts":"1731408966537"} 2024-11-12T10:56:06,539 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T10:56:06,539 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T10:56:06,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6900 sec 2024-11-12T10:56:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-12T10:56:06,997 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:56:06,997 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T10:56:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:06,999 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-12T10:56:06,999 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:07,001 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:56:07,003 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/recovered.edits] 2024-11-12T10:56:07,004 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/bbe57ea12b0e455abdb0431d79dfb44d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/bbe57ea12b0e455abdb0431d79dfb44d 2024-11-12T10:56:07,006 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/da63b95b898c4dc0a56c5c55d5ddbca7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/A/da63b95b898c4dc0a56c5c55d5ddbca7 2024-11-12T10:56:07,007 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/1a7ca16a311145f2834c99adc18691c1 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/1a7ca16a311145f2834c99adc18691c1 2024-11-12T10:56:07,008 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/a7f679b37a5a422b972537be1febf579 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/B/a7f679b37a5a422b972537be1febf579 2024-11-12T10:56:07,010 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/062f90d039324bd69a6e027b674fb513 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/062f90d039324bd69a6e027b674fb513 2024-11-12T10:56:07,011 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c2f5425cec2d43f69cd9a63a8b0ecb92 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/C/c2f5425cec2d43f69cd9a63a8b0ecb92 2024-11-12T10:56:07,013 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/recovered.edits/377.seqid to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997/recovered.edits/377.seqid 2024-11-12T10:56:07,014 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/5b5c37b8175d8b0adea03f092f215997 2024-11-12T10:56:07,014 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T10:56:07,016 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:07,018 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T10:56:07,020 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T10:56:07,021 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:07,021 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T10:56:07,021 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731408967021"}]},"ts":"9223372036854775807"} 2024-11-12T10:56:07,023 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T10:56:07,023 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5b5c37b8175d8b0adea03f092f215997, NAME => 'TestAcidGuarantees,,1731408940139.5b5c37b8175d8b0adea03f092f215997.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T10:56:07,023 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T10:56:07,023 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731408967023"}]},"ts":"9223372036854775807"} 2024-11-12T10:56:07,025 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T10:56:07,026 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:07,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-12T10:56:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-12T10:56:07,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T10:56:07,107 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:56:07,118 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252 (was 248) - Thread LEAK? -, OpenFileDescriptor=457 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=427 (was 488), ProcessCount=11 (was 11), AvailableMemoryMB=2315 (was 3075) 2024-11-12T10:56:07,130 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=427, ProcessCount=11, AvailableMemoryMB=2318 2024-11-12T10:56:07,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T10:56:07,131 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T10:56:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:07,133 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T10:56:07,134 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:07,134 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-12T10:56:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T10:56:07,135 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T10:56:07,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742338_1514 (size=1061) 2024-11-12T10:56:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T10:56:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T10:56:07,544 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98 2024-11-12T10:56:07,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742339_1515 (size=53) 2024-11-12T10:56:07,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T10:56:07,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:56:07,955 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c83771492407a48a02121732c6276538, disabling compactions & flushes 2024-11-12T10:56:07,955 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:07,955 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:07,955 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. after waiting 0 ms 2024-11-12T10:56:07,955 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:07,955 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:07,955 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c83771492407a48a02121732c6276538: Waiting for close lock at 1731408967954Disabling compacts and flushes for region at 1731408967954Disabling writes for close at 1731408967955 (+1 ms)Writing region close event to WAL at 1731408967955Closed at 1731408967955 2024-11-12T10:56:07,956 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T10:56:07,956 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731408967956"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731408967956"}]},"ts":"1731408967956"} 2024-11-12T10:56:07,957 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T10:56:07,958 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T10:56:07,958 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408967958"}]},"ts":"1731408967958"} 2024-11-12T10:56:07,959 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T10:56:07,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, ASSIGN}] 2024-11-12T10:56:07,960 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, ASSIGN 2024-11-12T10:56:07,961 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, ASSIGN; state=OFFLINE, location=aed202db7571,43143,1731408812834; forceNewPlan=false, retain=false 2024-11-12T10:56:08,112 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c83771492407a48a02121732c6276538, regionState=OPENING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:56:08,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, ASSIGN because future has completed 2024-11-12T10:56:08,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:56:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T10:56:08,274 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:08,274 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:56:08,275 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c83771492407a48a02121732c6276538 2024-11-12T10:56:08,275 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:56:08,275 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for c83771492407a48a02121732c6276538 2024-11-12T10:56:08,275 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for c83771492407a48a02121732c6276538 2024-11-12T10:56:08,276 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c83771492407a48a02121732c6276538 2024-11-12T10:56:08,278 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:56:08,278 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c83771492407a48a02121732c6276538 columnFamilyName A 2024-11-12T10:56:08,278 DEBUG [StoreOpener-c83771492407a48a02121732c6276538-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:08,279 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.HStore(327): Store=c83771492407a48a02121732c6276538/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:56:08,279 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c83771492407a48a02121732c6276538 2024-11-12T10:56:08,280 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:56:08,280 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c83771492407a48a02121732c6276538 columnFamilyName B 2024-11-12T10:56:08,280 DEBUG [StoreOpener-c83771492407a48a02121732c6276538-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:08,281 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.HStore(327): Store=c83771492407a48a02121732c6276538/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:56:08,281 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c83771492407a48a02121732c6276538 2024-11-12T10:56:08,282 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:56:08,282 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c83771492407a48a02121732c6276538 columnFamilyName C 2024-11-12T10:56:08,282 DEBUG [StoreOpener-c83771492407a48a02121732c6276538-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:08,283 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.HStore(327): Store=c83771492407a48a02121732c6276538/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:56:08,283 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:08,283 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for c83771492407a48a02121732c6276538 2024-11-12T10:56:08,284 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538 2024-11-12T10:56:08,284 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538 2024-11-12T10:56:08,285 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for c83771492407a48a02121732c6276538 2024-11-12T10:56:08,285 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for c83771492407a48a02121732c6276538 2024-11-12T10:56:08,285 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T10:56:08,287 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for c83771492407a48a02121732c6276538 2024-11-12T10:56:08,288 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:56:08,288 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened c83771492407a48a02121732c6276538; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72320001, jitterRate=0.0776519924402237}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T10:56:08,288 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c83771492407a48a02121732c6276538 2024-11-12T10:56:08,289 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for c83771492407a48a02121732c6276538: Running coprocessor pre-open hook at 1731408968275Writing region info on filesystem at 1731408968275Initializing all the Stores at 1731408968276 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408968276Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408968276Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408968276Cleaning up temporary data from old regions at 1731408968285 (+9 ms)Running coprocessor post-open hooks at 1731408968288 (+3 ms)Region opened successfully at 1731408968289 (+1 ms) 2024-11-12T10:56:08,290 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., pid=148, masterSystemTime=1731408968272 2024-11-12T10:56:08,291 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:08,291 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:08,292 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c83771492407a48a02121732c6276538, regionState=OPEN, openSeqNum=2, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:56:08,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:56:08,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-12T10:56:08,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 in 177 msec 2024-11-12T10:56:08,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-12T10:56:08,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, ASSIGN in 336 msec 2024-11-12T10:56:08,297 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T10:56:08,297 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408968297"}]},"ts":"1731408968297"} 2024-11-12T10:56:08,298 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T10:56:08,299 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T10:56:08,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1680 sec 2024-11-12T10:56:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T10:56:09,287 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:56:09,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145ad34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:09,288 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:56:09,288 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:56:09,289 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:56:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:56:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:56:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e89096c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:56:09,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:56:09,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:09,290 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:56:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56c2d7e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:09,291 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:56:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:56:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:56:09,293 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:56:09,293 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T10:56:09,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is aed202db7571,45901,1731408812240 2024-11-12T10:56:09,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@647a83c8 2024-11-12T10:56:09,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T10:56:09,295 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T10:56:09,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T10:56:09,296 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T10:56:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:09,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742340_1516 (size=1097) 2024-11-12T10:56:09,711 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T10:56:09,711 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T10:56:09,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T10:56:09,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, REOPEN/MOVE}] 2024-11-12T10:56:09,721 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, REOPEN/MOVE 2024-11-12T10:56:09,722 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=c83771492407a48a02121732c6276538, regionState=CLOSING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:56:09,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, REOPEN/MOVE because future has completed 2024-11-12T10:56:09,723 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T10:56:09,723 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:56:09,876 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close c83771492407a48a02121732c6276538 2024-11-12T10:56:09,876 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T10:56:09,876 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing c83771492407a48a02121732c6276538, disabling compactions & flushes 2024-11-12T10:56:09,876 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:09,876 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:09,876 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. after waiting 0 ms 2024-11-12T10:56:09,877 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:09,885 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T10:56:09,886 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:09,886 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for c83771492407a48a02121732c6276538: Waiting for close lock at 1731408969876Running coprocessor pre-close hooks at 1731408969876Disabling compacts and flushes for region at 1731408969876Disabling writes for close at 1731408969876Writing region close event to WAL at 1731408969877 (+1 ms)Running coprocessor post-close hooks at 1731408969886 (+9 ms)Closed at 1731408969886 2024-11-12T10:56:09,886 WARN [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: c83771492407a48a02121732c6276538 to self. 2024-11-12T10:56:09,890 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed c83771492407a48a02121732c6276538 2024-11-12T10:56:09,891 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=c83771492407a48a02121732c6276538, regionState=CLOSED 2024-11-12T10:56:09,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:56:09,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-12T10:56:09,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 in 172 msec 2024-11-12T10:56:09,899 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, REOPEN/MOVE; state=CLOSED, location=aed202db7571,43143,1731408812834; forceNewPlan=false, retain=true 2024-11-12T10:56:10,050 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=c83771492407a48a02121732c6276538, regionState=OPENING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, REOPEN/MOVE because future has completed 2024-11-12T10:56:10,052 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:56:10,211 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:10,211 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:56:10,212 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c83771492407a48a02121732c6276538 2024-11-12T10:56:10,212 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:56:10,212 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for c83771492407a48a02121732c6276538 2024-11-12T10:56:10,212 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for c83771492407a48a02121732c6276538 2024-11-12T10:56:10,215 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c83771492407a48a02121732c6276538 2024-11-12T10:56:10,216 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:56:10,217 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c83771492407a48a02121732c6276538 columnFamilyName A 2024-11-12T10:56:10,218 DEBUG [StoreOpener-c83771492407a48a02121732c6276538-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:10,218 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.HStore(327): Store=c83771492407a48a02121732c6276538/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:56:10,218 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c83771492407a48a02121732c6276538 2024-11-12T10:56:10,219 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:56:10,219 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c83771492407a48a02121732c6276538 columnFamilyName B 2024-11-12T10:56:10,219 DEBUG [StoreOpener-c83771492407a48a02121732c6276538-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:10,219 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.HStore(327): Store=c83771492407a48a02121732c6276538/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:56:10,220 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c83771492407a48a02121732c6276538 2024-11-12T10:56:10,220 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T10:56:10,220 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c83771492407a48a02121732c6276538 columnFamilyName C 2024-11-12T10:56:10,220 DEBUG [StoreOpener-c83771492407a48a02121732c6276538-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:10,221 INFO [StoreOpener-c83771492407a48a02121732c6276538-1 {}] regionserver.HStore(327): Store=c83771492407a48a02121732c6276538/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:56:10,221 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:10,221 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for c83771492407a48a02121732c6276538 2024-11-12T10:56:10,221 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538 2024-11-12T10:56:10,222 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538 2024-11-12T10:56:10,223 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for c83771492407a48a02121732c6276538 2024-11-12T10:56:10,223 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for c83771492407a48a02121732c6276538 2024-11-12T10:56:10,223 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T10:56:10,225 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for c83771492407a48a02121732c6276538 2024-11-12T10:56:10,225 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened c83771492407a48a02121732c6276538; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59262471, jitterRate=-0.1169203668832779}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T10:56:10,225 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c83771492407a48a02121732c6276538 2024-11-12T10:56:10,226 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for c83771492407a48a02121732c6276538: Running coprocessor pre-open hook at 1731408970213Writing region info on filesystem at 1731408970213Initializing all the Stores at 1731408970214 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408970214Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408970215 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731408970215Cleaning up temporary data from old regions at 1731408970223 (+8 ms)Running coprocessor post-open hooks at 1731408970225 (+2 ms)Region opened successfully at 1731408970226 (+1 ms) 2024-11-12T10:56:10,227 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., pid=153, masterSystemTime=1731408970204 2024-11-12T10:56:10,228 DEBUG [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:10,228 INFO [RS_OPEN_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:10,229 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=c83771492407a48a02121732c6276538, regionState=OPEN, openSeqNum=5, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:56:10,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-12T10:56:10,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 in 179 msec 2024-11-12T10:56:10,235 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-12T10:56:10,235 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, REOPEN/MOVE in 512 msec 2024-11-12T10:56:10,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-12T10:56:10,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 517 msec 2024-11-12T10:56:10,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-11-12T10:56:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-12T10:56:10,240 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:56:10,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:56:10,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:56:10,241 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:56:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:56:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:56:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cc4a696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:56:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:56:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:10,243 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:56:10,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45341de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:56:10,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:56:10,245 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:56:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:56:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:56:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53dca9fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:56:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:56:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:10,247 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:56:10,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ec46d4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,248 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:56:10,248 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:56:10,249 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:56:10,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:56:10,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:56:10,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ca5a5c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:56:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:56:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:10,251 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:56:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d6822e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:56:10,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:56:10,253 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:56:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:56:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:56:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b4db9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:56:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:56:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:10,255 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:56:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aca35d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:56:10,257 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:56:10,257 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:56:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:56:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:56:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a408aba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:56:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:56:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:10,259 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:56:10,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f69877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:56:10,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:56:10,261 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:56:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:56:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:56:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0b9b7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:56:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:56:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:10,262 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:56:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15379c76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:56:10,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:56:10,264 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:56:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:56:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:56:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d693487, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:56:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:56:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:10,265 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:56:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5e3c7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,266 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:56:10,266 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:56:10,267 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:56:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:56:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:56:10,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a99b8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:56:10,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:56:10,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:10,269 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:56:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af8e69e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,270 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:56:10,270 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:56:10,271 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:56:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:56:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:56:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d961391, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:56:10,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:56:10,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:10,272 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:56:10,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39967b27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b93642c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aed202db7571,45901,-1 for getting cluster id 2024-11-12T10:56:10,274 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:56:10,274 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c5c7f0a-4e23-4f9f-b145-59d201be2441' 2024-11-12T10:56:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:56:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c5c7f0a-4e23-4f9f-b145-59d201be2441" 2024-11-12T10:56:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43217711, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aed202db7571,45901,-1] 2024-11-12T10:56:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:56:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:10,275 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:56:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6474977d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:56:10,276 DEBUG [Thread-2330 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:56:10,276 DEBUG [Thread-2332 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:56:10,276 DEBUG [Thread-2319 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:56:10,276 DEBUG [Thread-2323 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:56:10,276 DEBUG [Thread-2334 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:56:10,276 DEBUG [Thread-2321 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:56:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:56:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:56:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:56:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:56:10,277 DEBUG [Thread-2325 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:56:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:56:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:56:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:56:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:56:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:56:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:56:10,278 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:56:10,278 DEBUG [Thread-2336 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:56:10,278 DEBUG [Thread-2328 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:56:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:56:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:56:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:56:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:56:10,278 DEBUG [Thread-2317 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:56:10,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:56:10,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:56:10,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:56:10,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:56:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-12T10:56:10,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:56:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:56:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:56:10,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:56:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:56:10,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:56:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:56:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T10:56:10,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:56:10,280 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:56:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aed202db7571,43143,1731408812834, seqNum=-1] 2024-11-12T10:56:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:56:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:56:10,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:56:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:56:10,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:56:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:56:10,283 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:56:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:56:10,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:56:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:56:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:56:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:56:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:56:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:56:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5] 2024-11-12T10:56:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on c83771492407a48a02121732c6276538 2024-11-12T10:56:10,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T10:56:10,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:10,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:10,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:10,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:10,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:10,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:10,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731408970299, max timestamp=1731408970316Num uniques 859; 2024-11-12T10:56:10,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T10:56:10,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408970299, max timestamp=1731408970318Num uniques 859; 2024-11-12T10:56:10,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:56:10,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408970299, max timestamp=1731408970318Num uniques 859; 2024-11-12T10:56:10,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:56:10,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ec1594b6e8dc4a16b469688f3ed29b87_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408970289/Put/seqid=0 2024-11-12T10:56:10,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742341_1517 (size=12154) 2024-11-12T10:56:10,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409030338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1731408970335, max timestamp=1731408970343Num uniques 1068; 2024-11-12T10:56:10,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1731408970335, max timestamp=1731408970343Num uniques 1018; 2024-11-12T10:56:10,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48898, new segment data size=9968, suffix heap size=139040,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T10:56:10,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46608, new segment data size=9968, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T10:56:10,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731408970335, max timestamp=1731408970343Num uniques 1018; 2024-11-12T10:56:10,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46611, new segment data size=6870, suffix heap size=133840,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-12T10:56:10,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37572 deadline: 1731409030366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37608 deadline: 1731409030367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409030369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409030369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T10:56:10,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T10:56:10,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:10,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:10,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:10,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:10,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:10,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409030447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37572 deadline: 1731409030477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37608 deadline: 1731409030477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409030477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409030477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T10:56:10,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:10,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:10,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:10,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:10,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T10:56:10,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409030657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37572 deadline: 1731409030687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37608 deadline: 1731409030687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409030688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409030688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,736 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:10,739 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ec1594b6e8dc4a16b469688f3ed29b87_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ec1594b6e8dc4a16b469688f3ed29b87_c83771492407a48a02121732c6276538 2024-11-12T10:56:10,739 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/9dcf58b0f5294d668e067d3c609c263a, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:10,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/9dcf58b0f5294d668e067d3c609c263a is 175, key is test_row_0/A:col10/1731408970289/Put/seqid=0 2024-11-12T10:56:10,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742342_1518 (size=30955) 2024-11-12T10:56:10,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T10:56:10,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:10,743 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/9dcf58b0f5294d668e067d3c609c263a 2024-11-12T10:56:10,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:10,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:10,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:10,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:10,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:10,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/f668d534bcf94feaa3cb6d45d184288c is 50, key is test_row_0/B:col10/1731408970289/Put/seqid=0 2024-11-12T10:56:10,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742343_1519 (size=12001) 2024-11-12T10:56:10,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T10:56:10,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:10,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:10,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:10,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:10,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T10:56:10,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409030967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:10,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37572 deadline: 1731409030997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:11,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37608 deadline: 1731409030997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:11,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409030997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:11,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409030998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:11,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T10:56:11,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:11,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:11,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:11,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:11,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:11,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/f668d534bcf94feaa3cb6d45d184288c 2024-11-12T10:56:11,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/83a0c74214e148e9834a037a5860b843 is 50, key is test_row_0/C:col10/1731408970289/Put/seqid=0 2024-11-12T10:56:11,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742344_1520 (size=12001) 2024-11-12T10:56:11,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/83a0c74214e148e9834a037a5860b843 2024-11-12T10:56:11,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/9dcf58b0f5294d668e067d3c609c263a as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/9dcf58b0f5294d668e067d3c609c263a 2024-11-12T10:56:11,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/9dcf58b0f5294d668e067d3c609c263a, entries=150, sequenceid=14, filesize=30.2 K 2024-11-12T10:56:11,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/f668d534bcf94feaa3cb6d45d184288c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/f668d534bcf94feaa3cb6d45d184288c 2024-11-12T10:56:11,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/f668d534bcf94feaa3cb6d45d184288c, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T10:56:11,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/83a0c74214e148e9834a037a5860b843 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/83a0c74214e148e9834a037a5860b843 2024-11-12T10:56:11,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T10:56:11,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:11,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:11,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:11,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:11,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:11,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/83a0c74214e148e9834a037a5860b843, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T10:56:11,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=137.86 KB/141169 for c83771492407a48a02121732c6276538 in 912ms, sequenceid=14, compaction requested=false 2024-11-12T10:56:11,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:11,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T10:56:11,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:11,353 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=137.86 KB heapSize=392.47 KB 2024-11-12T10:56:11,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:11,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:56:11,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:11,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:56:11,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:11,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:56:11,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e69642c76c1143bea55c4c2ddfd4221d_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408970364/Put/seqid=0 2024-11-12T10:56:11,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742345_1521 (size=12154) 2024-11-12T10:56:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T10:56:11,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on c83771492407a48a02121732c6276538 2024-11-12T10:56:11,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409031503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:11,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37572 deadline: 1731409031507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409031507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37608 deadline: 1731409031507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409031507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:11,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:11,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409031617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:11,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:11,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:11,766 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e69642c76c1143bea55c4c2ddfd4221d_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e69642c76c1143bea55c4c2ddfd4221d_c83771492407a48a02121732c6276538 2024-11-12T10:56:11,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/6c181619613542bca0dd45a66699bf52, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:11,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/6c181619613542bca0dd45a66699bf52 is 175, key is test_row_0/A:col10/1731408970364/Put/seqid=0 2024-11-12T10:56:11,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742346_1522 (size=30955) 2024-11-12T10:56:11,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409031826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:11,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:11,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:12,045 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T10:56:12,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409032137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:12,172 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/6c181619613542bca0dd45a66699bf52 2024-11-12T10:56:12,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/513e5800f3bb49529ce1e0179f5f1b16 is 50, key is test_row_0/B:col10/1731408970364/Put/seqid=0 2024-11-12T10:56:12,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742347_1523 (size=12001) 2024-11-12T10:56:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T10:56:12,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37572 deadline: 1731409032517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:12,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409032517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:12,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:12,520 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409032517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:12,520 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:12,520 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:12,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37608 deadline: 1731409032527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:12,539 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:12,581 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/513e5800f3bb49529ce1e0179f5f1b16 2024-11-12T10:56:12,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/b75e4ab6bb3e46afa3d726a83aea9853 is 50, key is test_row_0/C:col10/1731408970364/Put/seqid=0 2024-11-12T10:56:12,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742348_1524 (size=12001) 2024-11-12T10:56:12,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409032647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:12,992 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.93 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/b75e4ab6bb3e46afa3d726a83aea9853 2024-11-12T10:56:13,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/6c181619613542bca0dd45a66699bf52 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/6c181619613542bca0dd45a66699bf52 2024-11-12T10:56:13,009 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/6c181619613542bca0dd45a66699bf52, entries=150, sequenceid=68, filesize=30.2 K 2024-11-12T10:56:13,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/513e5800f3bb49529ce1e0179f5f1b16 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/513e5800f3bb49529ce1e0179f5f1b16 2024-11-12T10:56:13,017 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/513e5800f3bb49529ce1e0179f5f1b16, entries=150, sequenceid=68, filesize=11.7 K 2024-11-12T10:56:13,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/b75e4ab6bb3e46afa3d726a83aea9853 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/b75e4ab6bb3e46afa3d726a83aea9853 2024-11-12T10:56:13,024 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/b75e4ab6bb3e46afa3d726a83aea9853, entries=150, sequenceid=68, filesize=11.7 K 2024-11-12T10:56:13,025 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~137.86 KB/141169, heapSize ~392.42 KB/401840, currentSize=40.25 KB/41220 for c83771492407a48a02121732c6276538 in 1672ms, sequenceid=68, compaction requested=false 2024-11-12T10:56:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-12T10:56:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-12T10:56:13,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-12T10:56:13,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7460 sec 2024-11-12T10:56:13,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.7540 sec 2024-11-12T10:56:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on c83771492407a48a02121732c6276538 2024-11-12T10:56:13,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T10:56:13,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:13,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:13,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:13,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:13,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:13,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:13,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126577305f5ef7449ebf94e9176cca7c95_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408971492/Put/seqid=0 2024-11-12T10:56:13,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742349_1525 (size=12154) 2024-11-12T10:56:13,705 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:13,707 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126577305f5ef7449ebf94e9176cca7c95_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126577305f5ef7449ebf94e9176cca7c95_c83771492407a48a02121732c6276538 2024-11-12T10:56:13,708 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/564ecda3829c4356aea5f5caa17bc9fb, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:13,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/564ecda3829c4356aea5f5caa17bc9fb is 175, key is test_row_0/A:col10/1731408971492/Put/seqid=0 2024-11-12T10:56:13,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742350_1526 (size=30955) 2024-11-12T10:56:13,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408973698, max timestamp=1731408973717Num uniques 859; 2024-11-12T10:56:13,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408973698, max timestamp=1731408973717Num uniques 859; 2024-11-12T10:56:13,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408973698, max timestamp=1731408973717Num uniques 859; 2024-11-12T10:56:13,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:56:13,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:56:13,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:56:13,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:13,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409033799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:13,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409033907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:13,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:13,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:13,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:14,112 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/564ecda3829c4356aea5f5caa17bc9fb 2024-11-12T10:56:14,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/29871735f5564523bf74058b7350cb2b is 50, key is test_row_0/B:col10/1731408971492/Put/seqid=0 2024-11-12T10:56:14,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409034117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:14,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742351_1527 (size=12001) 2024-11-12T10:56:14,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409034427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T10:56:14,447 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:56:14,447 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:56:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-12T10:56:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T10:56:14,449 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:56:14,449 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:56:14,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:56:14,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/29871735f5564523bf74058b7350cb2b 2024-11-12T10:56:14,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/4271a256876f448fbee740ce5c239be5 is 50, key is test_row_0/C:col10/1731408971492/Put/seqid=0 2024-11-12T10:56:14,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742352_1528 (size=12001) 2024-11-12T10:56:14,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37572 deadline: 1731409034527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:14,529 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:14,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409034536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:14,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409034537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:14,538 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_1' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:14,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:14,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:14,538 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T10:56:14,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37608 deadline: 1731409034567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:14,569 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:14,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:14,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:14,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:14,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:14,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:14,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:14,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:14,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:14,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:14,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:14,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:14,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:14,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:14,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:14,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T10:56:14,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:14,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:14,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:14,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:14,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:14,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:14,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/4271a256876f448fbee740ce5c239be5 2024-11-12T10:56:14,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/564ecda3829c4356aea5f5caa17bc9fb as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/564ecda3829c4356aea5f5caa17bc9fb 2024-11-12T10:56:14,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/564ecda3829c4356aea5f5caa17bc9fb, entries=150, sequenceid=78, filesize=30.2 K 2024-11-12T10:56:14,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/29871735f5564523bf74058b7350cb2b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/29871735f5564523bf74058b7350cb2b 2024-11-12T10:56:14,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/29871735f5564523bf74058b7350cb2b, entries=150, sequenceid=78, filesize=11.7 K 2024-11-12T10:56:14,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/4271a256876f448fbee740ce5c239be5 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/4271a256876f448fbee740ce5c239be5 2024-11-12T10:56:14,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/4271a256876f448fbee740ce5c239be5, entries=150, sequenceid=78, filesize=11.7 K 2024-11-12T10:56:14,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c83771492407a48a02121732c6276538 in 1280ms, sequenceid=78, compaction requested=true 2024-11-12T10:56:14,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:14,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:56:14,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:14,941 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:14,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:56:14,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:14,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:56:14,941 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:14,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:14,942 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:14,942 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:14,942 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/A is initiating minor compaction (all files) 2024-11-12T10:56:14,942 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/B is initiating minor compaction (all files) 2024-11-12T10:56:14,942 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/B in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:14,942 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/A in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on c83771492407a48a02121732c6276538 2024-11-12T10:56:14,942 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/f668d534bcf94feaa3cb6d45d184288c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/513e5800f3bb49529ce1e0179f5f1b16, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/29871735f5564523bf74058b7350cb2b] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=35.2 K 2024-11-12T10:56:14,942 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/9dcf58b0f5294d668e067d3c609c263a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/6c181619613542bca0dd45a66699bf52, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/564ecda3829c4356aea5f5caa17bc9fb] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=90.7 K 2024-11-12T10:56:14,942 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:14,942 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/9dcf58b0f5294d668e067d3c609c263a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/6c181619613542bca0dd45a66699bf52, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/564ecda3829c4356aea5f5caa17bc9fb] 2024-11-12T10:56:14,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-12T10:56:14,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:14,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T10:56:14,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:14,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T10:56:14,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:14,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T10:56:14,945 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting f668d534bcf94feaa3cb6d45d184288c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731408970282 2024-11-12T10:56:14,945 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dcf58b0f5294d668e067d3c609c263a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731408970282 2024-11-12T10:56:14,946 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 513e5800f3bb49529ce1e0179f5f1b16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731408970364 2024-11-12T10:56:14,946 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c181619613542bca0dd45a66699bf52, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731408970364 2024-11-12T10:56:14,946 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 29871735f5564523bf74058b7350cb2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731408971488 2024-11-12T10:56:14,946 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 564ecda3829c4356aea5f5caa17bc9fb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731408971488 2024-11-12T10:56:14,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111295a956a59cc74f2e9d799ee5e0cfb380_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408973797/Put/seqid=0 2024-11-12T10:56:14,953 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:14,955 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#B#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:14,955 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/94534cf3993340f6bf2624d4412fc8a2 is 50, key is test_row_0/B:col10/1731408971492/Put/seqid=0 2024-11-12T10:56:14,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742353_1529 (size=14594) 2024-11-12T10:56:14,958 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111265ee4bba98ab457cbb610217ef72eb58_c83771492407a48a02121732c6276538 store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:14,959 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111265ee4bba98ab457cbb610217ef72eb58_c83771492407a48a02121732c6276538, store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:14,959 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111265ee4bba98ab457cbb610217ef72eb58_c83771492407a48a02121732c6276538 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:14,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742354_1530 (size=12104) 2024-11-12T10:56:14,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742355_1531 (size=4469) 2024-11-12T10:56:14,963 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#A#compaction#456 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:14,963 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/b90573e9cb534bb3967ddc16eacd78a8 is 175, key is test_row_0/A:col10/1731408971492/Put/seqid=0 2024-11-12T10:56:14,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742356_1532 (size=31058) 2024-11-12T10:56:14,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409034973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:15,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:15,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:15,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T10:56:15,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409035087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:15,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:15,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409035297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:15,358 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:15,362 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111295a956a59cc74f2e9d799ee5e0cfb380_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111295a956a59cc74f2e9d799ee5e0cfb380_c83771492407a48a02121732c6276538 2024-11-12T10:56:15,362 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/830f3cceb14a485db9c26ad3f993e649, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:15,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/830f3cceb14a485db9c26ad3f993e649 is 175, key is test_row_0/A:col10/1731408973797/Put/seqid=0 2024-11-12T10:56:15,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:15,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:15,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742357_1533 (size=39549) 2024-11-12T10:56:15,370 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/94534cf3993340f6bf2624d4412fc8a2 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/94534cf3993340f6bf2624d4412fc8a2 2024-11-12T10:56:15,371 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/b90573e9cb534bb3967ddc16eacd78a8 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b90573e9cb534bb3967ddc16eacd78a8 2024-11-12T10:56:15,374 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/A of c83771492407a48a02121732c6276538 into b90573e9cb534bb3967ddc16eacd78a8(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:15,374 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:15,374 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/A, priority=13, startTime=1731408974941; duration=0sec 2024-11-12T10:56:15,374 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:15,375 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:A 2024-11-12T10:56:15,375 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:15,375 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/B of c83771492407a48a02121732c6276538 into 94534cf3993340f6bf2624d4412fc8a2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:15,375 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:15,375 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/B, priority=13, startTime=1731408974941; duration=0sec 2024-11-12T10:56:15,375 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:15,375 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:B 2024-11-12T10:56:15,375 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:15,375 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/C is initiating minor compaction (all files) 2024-11-12T10:56:15,375 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/C in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,375 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/83a0c74214e148e9834a037a5860b843, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/b75e4ab6bb3e46afa3d726a83aea9853, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/4271a256876f448fbee740ce5c239be5] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=35.2 K 2024-11-12T10:56:15,376 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83a0c74214e148e9834a037a5860b843, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731408970282 2024-11-12T10:56:15,376 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting b75e4ab6bb3e46afa3d726a83aea9853, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731408970364 2024-11-12T10:56:15,376 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4271a256876f448fbee740ce5c239be5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731408971488 2024-11-12T10:56:15,382 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#C#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:15,382 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/08523b6a4b47438f903acacae65ea8d7 is 50, key is test_row_0/C:col10/1731408971492/Put/seqid=0 2024-11-12T10:56:15,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742358_1534 (size=12104) 2024-11-12T10:56:15,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:15,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:15,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T10:56:15,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409035607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:15,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:15,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:15,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,768 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/830f3cceb14a485db9c26ad3f993e649 2024-11-12T10:56:15,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/7b5d226d50e54cd9af1d7b5c2ea65c13 is 50, key is test_row_0/B:col10/1731408973797/Put/seqid=0 2024-11-12T10:56:15,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742359_1535 (size=12001) 2024-11-12T10:56:15,794 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/08523b6a4b47438f903acacae65ea8d7 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/08523b6a4b47438f903acacae65ea8d7 2024-11-12T10:56:15,798 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/C of c83771492407a48a02121732c6276538 into 08523b6a4b47438f903acacae65ea8d7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:15,798 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:15,798 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/C, priority=13, startTime=1731408974941; duration=0sec 2024-11-12T10:56:15,798 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:15,798 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:C 2024-11-12T10:56:15,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:15,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:15,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:15,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:15,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:15,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:16,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409036117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:16,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:16,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:16,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:16,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:16,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:16,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:16,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:16,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/7b5d226d50e54cd9af1d7b5c2ea65c13 2024-11-12T10:56:16,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/3ec5d60cf58246ca8cddaaabe2cf6b43 is 50, key is test_row_0/C:col10/1731408973797/Put/seqid=0 2024-11-12T10:56:16,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742360_1536 (size=12001) 2024-11-12T10:56:16,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:16,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:16,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:16,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:16,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:16,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:16,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:16,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:16,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:16,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:16,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:16,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:16,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:16,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:16,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:16,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:16,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:16,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:16,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/3ec5d60cf58246ca8cddaaabe2cf6b43 2024-11-12T10:56:16,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/830f3cceb14a485db9c26ad3f993e649 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/830f3cceb14a485db9c26ad3f993e649 2024-11-12T10:56:16,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/830f3cceb14a485db9c26ad3f993e649, entries=200, sequenceid=116, filesize=38.6 K 2024-11-12T10:56:16,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/7b5d226d50e54cd9af1d7b5c2ea65c13 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/7b5d226d50e54cd9af1d7b5c2ea65c13 2024-11-12T10:56:16,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/7b5d226d50e54cd9af1d7b5c2ea65c13, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T10:56:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T10:56:16,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/3ec5d60cf58246ca8cddaaabe2cf6b43 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/3ec5d60cf58246ca8cddaaabe2cf6b43 2024-11-12T10:56:16,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/3ec5d60cf58246ca8cddaaabe2cf6b43, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T10:56:16,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for c83771492407a48a02121732c6276538 in 1657ms, sequenceid=116, compaction requested=false 2024-11-12T10:56:16,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:16,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T10:56:16,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:16,737 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-12T10:56:16,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:16,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:16,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:16,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:16,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:16,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:16,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111229f858d95a1c45f7a75447eed3dd3219_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408974972/Put/seqid=0 2024-11-12T10:56:16,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742361_1537 (size=12154) 2024-11-12T10:56:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on c83771492407a48a02121732c6276538 2024-11-12T10:56:17,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:17,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:17,152 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111229f858d95a1c45f7a75447eed3dd3219_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111229f858d95a1c45f7a75447eed3dd3219_c83771492407a48a02121732c6276538 2024-11-12T10:56:17,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/46861e40332642e396a3a8293241f4f3, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:17,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/46861e40332642e396a3a8293241f4f3 is 175, key is test_row_0/A:col10/1731408974972/Put/seqid=0 2024-11-12T10:56:17,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742362_1538 (size=30955) 2024-11-12T10:56:17,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408977183, max timestamp=1731408977200Num uniques 859; 2024-11-12T10:56:17,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408977183, max timestamp=1731408977200Num uniques 859; 2024-11-12T10:56:17,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408977183, max timestamp=1731408977200Num uniques 859; 2024-11-12T10:56:17,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:56:17,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:56:17,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:56:17,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731408977255, max timestamp=1731408977290Num uniques 1018; 2024-11-12T10:56:17,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T10:56:17,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731408977255, max timestamp=1731408977290Num uniques 1018; 2024-11-12T10:56:17,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731408977255, max timestamp=1731408977290Num uniques 1018; 2024-11-12T10:56:17,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T10:56:17,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T10:56:17,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731408977359, max timestamp=1731408977375Num uniques 1027; 2024-11-12T10:56:17,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-12T10:56:17,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731408977359, max timestamp=1731408977375Num uniques 1027; 2024-11-12T10:56:17,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731408977359, max timestamp=1731408977375Num uniques 1027; 2024-11-12T10:56:17,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-12T10:56:17,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-12T10:56:17,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731408977388, max timestamp=1731408977451Num uniques 1036; 2024-11-12T10:56:17,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-12T10:56:17,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731408977388, max timestamp=1731408977451Num uniques 1036; 2024-11-12T10:56:17,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-12T10:56:17,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731408977388, max timestamp=1731408977451Num uniques 1036; 2024-11-12T10:56:17,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-12T10:56:17,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731408977504, max timestamp=1731408977535Num uniques 1045; 2024-11-12T10:56:17,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-12T10:56:17,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731408977504, max timestamp=1731408977535Num uniques 1045; 2024-11-12T10:56:17,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-12T10:56:17,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731408977504, max timestamp=1731408977535Num uniques 1045; 2024-11-12T10:56:17,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-12T10:56:17,558 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/46861e40332642e396a3a8293241f4f3 2024-11-12T10:56:17,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/c3b2aaaed1734022826db97c1ee5d961 is 50, key is test_row_0/B:col10/1731408974972/Put/seqid=0 2024-11-12T10:56:17,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742363_1539 (size=12001) 2024-11-12T10:56:17,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409037623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:17,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409037737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:17,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409037946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:17,966 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/c3b2aaaed1734022826db97c1ee5d961 2024-11-12T10:56:17,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/92669762035b45b59b7b89dbd751b171 is 50, key is test_row_0/C:col10/1731408974972/Put/seqid=0 2024-11-12T10:56:17,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742364_1540 (size=12001) 2024-11-12T10:56:18,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409038257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:18,375 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/92669762035b45b59b7b89dbd751b171 2024-11-12T10:56:18,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/46861e40332642e396a3a8293241f4f3 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/46861e40332642e396a3a8293241f4f3 2024-11-12T10:56:18,380 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/46861e40332642e396a3a8293241f4f3, entries=150, sequenceid=128, filesize=30.2 K 2024-11-12T10:56:18,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/c3b2aaaed1734022826db97c1ee5d961 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/c3b2aaaed1734022826db97c1ee5d961 2024-11-12T10:56:18,383 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/c3b2aaaed1734022826db97c1ee5d961, entries=150, sequenceid=128, filesize=11.7 K 2024-11-12T10:56:18,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/92669762035b45b59b7b89dbd751b171 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/92669762035b45b59b7b89dbd751b171 2024-11-12T10:56:18,386 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/92669762035b45b59b7b89dbd751b171, entries=150, sequenceid=128, filesize=11.7 K 2024-11-12T10:56:18,387 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for c83771492407a48a02121732c6276538 in 1650ms, sequenceid=128, compaction requested=true 2024-11-12T10:56:18,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:18,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:18,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-12T10:56:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-12T10:56:18,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-12T10:56:18,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9390 sec 2024-11-12T10:56:18,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 3.9430 sec 2024-11-12T10:56:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on c83771492407a48a02121732c6276538 2024-11-12T10:56:18,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=147.60 KB heapSize=422.16 KB 2024-11-12T10:56:18,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:18,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T10:56:18,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:18,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T10:56:18,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:18,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T10:56:18,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126e230ac3c49e46719fd6200e8b6cde90_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408977596/Put/seqid=0 2024-11-12T10:56:18,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742365_1541 (size=14794) 2024-11-12T10:56:18,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409038567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:18,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409038568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:18,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:18,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37572 deadline: 1731409038577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:18,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37608 deadline: 1731409038577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:18,581 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:18,581 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T10:56:18,617 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:56:18,618 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:56:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-12T10:56:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T10:56:18,619 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:56:18,620 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:56:18,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:56:18,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409038676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:18,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409038676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T10:56:18,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409038767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:18,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:18,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T10:56:18,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:18,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:18,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:18,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:18,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:18,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409038887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409038887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:18,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T10:56:18,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:18,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:18,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:18,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:18,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T10:56:18,961 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:18,964 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126e230ac3c49e46719fd6200e8b6cde90_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126e230ac3c49e46719fd6200e8b6cde90_c83771492407a48a02121732c6276538 2024-11-12T10:56:18,964 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/e866473a8d35466dae898c18557c2b16, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:18,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/e866473a8d35466dae898c18557c2b16 is 175, key is test_row_0/A:col10/1731408977596/Put/seqid=0 2024-11-12T10:56:18,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742366_1542 (size=39749) 2024-11-12T10:56:19,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T10:56:19,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:19,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:19,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:19,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409039197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409039197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:19,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T10:56:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:19,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T10:56:19,368 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/e866473a8d35466dae898c18557c2b16 2024-11-12T10:56:19,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/a573678afa52478cba0d4615a13ce72b is 50, key is test_row_0/B:col10/1731408977596/Put/seqid=0 2024-11-12T10:56:19,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742367_1543 (size=12151) 2024-11-12T10:56:19,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T10:56:19,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:19,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:19,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:19,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T10:56:19,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:19,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:19,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:19,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T10:56:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:19,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409039707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:19,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409039707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T10:56:19,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/a573678afa52478cba0d4615a13ce72b 2024-11-12T10:56:19,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/7a100265893f4dafbc7a492ad61f46d0 is 50, key is test_row_0/C:col10/1731408977596/Put/seqid=0 2024-11-12T10:56:19,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742368_1544 (size=12151) 2024-11-12T10:56:19,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409039787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:19,791 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:19,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T10:56:19,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:19,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:19,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:19,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T10:56:19,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:19,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:19,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:19,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:20,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T10:56:20,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:20,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:20,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:20,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:20,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:20,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/7a100265893f4dafbc7a492ad61f46d0 2024-11-12T10:56:20,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/e866473a8d35466dae898c18557c2b16 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/e866473a8d35466dae898c18557c2b16 2024-11-12T10:56:20,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/e866473a8d35466dae898c18557c2b16, entries=200, sequenceid=235, filesize=38.8 K 2024-11-12T10:56:20,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/a573678afa52478cba0d4615a13ce72b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/a573678afa52478cba0d4615a13ce72b 2024-11-12T10:56:20,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/a573678afa52478cba0d4615a13ce72b, entries=150, sequenceid=235, filesize=11.9 K 2024-11-12T10:56:20,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/7a100265893f4dafbc7a492ad61f46d0 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/7a100265893f4dafbc7a492ad61f46d0 2024-11-12T10:56:20,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/7a100265893f4dafbc7a492ad61f46d0, entries=150, sequenceid=235, filesize=11.9 K 2024-11-12T10:56:20,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~422.11 KB/432240, currentSize=33.54 KB/34350 for c83771492407a48a02121732c6276538 in 1650ms, sequenceid=235, compaction requested=true 2024-11-12T10:56:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:56:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:20,201 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:56:20,201 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:56:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:56:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:56:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:20,202 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:56:20,202 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141311 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:56:20,202 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/A is initiating minor compaction (all files) 2024-11-12T10:56:20,202 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/B is initiating minor compaction (all files) 2024-11-12T10:56:20,202 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/B in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:20,202 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/94534cf3993340f6bf2624d4412fc8a2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/7b5d226d50e54cd9af1d7b5c2ea65c13, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/c3b2aaaed1734022826db97c1ee5d961, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/a573678afa52478cba0d4615a13ce72b] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=47.1 K 2024-11-12T10:56:20,203 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/A in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:20,203 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b90573e9cb534bb3967ddc16eacd78a8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/830f3cceb14a485db9c26ad3f993e649, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/46861e40332642e396a3a8293241f4f3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/e866473a8d35466dae898c18557c2b16] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=138.0 K 2024-11-12T10:56:20,203 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:20,203 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b90573e9cb534bb3967ddc16eacd78a8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/830f3cceb14a485db9c26ad3f993e649, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/46861e40332642e396a3a8293241f4f3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/e866473a8d35466dae898c18557c2b16] 2024-11-12T10:56:20,204 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 94534cf3993340f6bf2624d4412fc8a2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731408971488 2024-11-12T10:56:20,204 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting b90573e9cb534bb3967ddc16eacd78a8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731408971488 2024-11-12T10:56:20,204 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b5d226d50e54cd9af1d7b5c2ea65c13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731408973792 2024-11-12T10:56:20,204 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 830f3cceb14a485db9c26ad3f993e649, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731408973754 2024-11-12T10:56:20,204 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b2aaaed1734022826db97c1ee5d961, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731408974947 2024-11-12T10:56:20,204 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46861e40332642e396a3a8293241f4f3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731408974947 2024-11-12T10:56:20,204 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting a573678afa52478cba0d4615a13ce72b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731408977586 2024-11-12T10:56:20,204 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting e866473a8d35466dae898c18557c2b16, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731408977586 2024-11-12T10:56:20,211 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#B#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:20,212 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/f818c9de68c246a194e39d5d2df8744b is 50, key is test_row_0/B:col10/1731408977596/Put/seqid=0 2024-11-12T10:56:20,218 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:20,219 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112191ce67d966843a1859e9b34f146a7c7_c83771492407a48a02121732c6276538 store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:20,221 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112191ce67d966843a1859e9b34f146a7c7_c83771492407a48a02121732c6276538, store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:20,221 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112191ce67d966843a1859e9b34f146a7c7_c83771492407a48a02121732c6276538 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:20,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742369_1545 (size=12391) 2024-11-12T10:56:20,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742370_1546 (size=4469) 2024-11-12T10:56:20,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T10:56:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:20,297 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T10:56:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:20,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112874a9354f4d547eeaa3048e9c0673c86_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408978561/Put/seqid=0 2024-11-12T10:56:20,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742371_1547 (size=12304) 2024-11-12T10:56:20,627 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/f818c9de68c246a194e39d5d2df8744b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/f818c9de68c246a194e39d5d2df8744b 2024-11-12T10:56:20,631 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c83771492407a48a02121732c6276538/B of c83771492407a48a02121732c6276538 into f818c9de68c246a194e39d5d2df8744b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:20,631 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:20,631 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/B, priority=12, startTime=1731408980201; duration=0sec 2024-11-12T10:56:20,631 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:20,631 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:B 2024-11-12T10:56:20,631 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:56:20,631 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T10:56:20,632 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/C is initiating minor compaction (all files) 2024-11-12T10:56:20,632 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/C in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:20,632 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/08523b6a4b47438f903acacae65ea8d7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/3ec5d60cf58246ca8cddaaabe2cf6b43, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/92669762035b45b59b7b89dbd751b171, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/7a100265893f4dafbc7a492ad61f46d0] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=47.1 K 2024-11-12T10:56:20,632 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 08523b6a4b47438f903acacae65ea8d7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731408971488 2024-11-12T10:56:20,632 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ec5d60cf58246ca8cddaaabe2cf6b43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731408973792 2024-11-12T10:56:20,632 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 92669762035b45b59b7b89dbd751b171, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731408974947 2024-11-12T10:56:20,633 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a100265893f4dafbc7a492ad61f46d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731408977586 2024-11-12T10:56:20,635 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#A#compaction#468 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:20,635 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/b6cfbffb1372481ca99eb4902e44bde2 is 175, key is test_row_0/A:col10/1731408977596/Put/seqid=0 2024-11-12T10:56:20,639 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#C#compaction#470 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:20,639 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/f62ee5777c7647bca4d2312cbd8fe1bd is 50, key is test_row_0/C:col10/1731408977596/Put/seqid=0 2024-11-12T10:56:20,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742372_1548 (size=31345) 2024-11-12T10:56:20,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742373_1549 (size=12391) 2024-11-12T10:56:20,644 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/b6cfbffb1372481ca99eb4902e44bde2 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b6cfbffb1372481ca99eb4902e44bde2 2024-11-12T10:56:20,648 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c83771492407a48a02121732c6276538/A of c83771492407a48a02121732c6276538 into b6cfbffb1372481ca99eb4902e44bde2(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:20,648 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:20,648 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/A, priority=12, startTime=1731408980201; duration=0sec 2024-11-12T10:56:20,648 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:20,648 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:A 2024-11-12T10:56:20,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:20,708 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112874a9354f4d547eeaa3048e9c0673c86_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112874a9354f4d547eeaa3048e9c0673c86_c83771492407a48a02121732c6276538 2024-11-12T10:56:20,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/539765ba6cc3427499abd8f7e4811675, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:20,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/539765ba6cc3427499abd8f7e4811675 is 175, key is test_row_0/A:col10/1731408978561/Put/seqid=0 2024-11-12T10:56:20,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742374_1550 (size=31105) 2024-11-12T10:56:20,712 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/539765ba6cc3427499abd8f7e4811675 2024-11-12T10:56:20,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/bc24a8cc674244938f1a7582bc2a64ca is 50, key is test_row_0/B:col10/1731408978561/Put/seqid=0 2024-11-12T10:56:20,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on c83771492407a48a02121732c6276538 2024-11-12T10:56:20,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742375_1551 (size=12151) 2024-11-12T10:56:20,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408980742, max timestamp=1731408980754Num uniques 859; 2024-11-12T10:56:20,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731408980742, max timestamp=1731408980754Num uniques 859; 2024-11-12T10:56:20,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T10:56:20,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:56:20,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731408980742, max timestamp=1731408980754Num uniques 859; 2024-11-12T10:56:20,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T10:56:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T10:56:20,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731408980785, max timestamp=1731408980800Num uniques 1018; 2024-11-12T10:56:20,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T10:56:20,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731408980785, max timestamp=1731408980800Num uniques 1068; 2024-11-12T10:56:20,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T10:56:20,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731408980785, max timestamp=1731408980800Num uniques 1018; 2024-11-12T10:56:20,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T10:56:20,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731408980825, max timestamp=1731408980842Num uniques 1077; 2024-11-12T10:56:20,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T10:56:20,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731408980825, max timestamp=1731408980842Num uniques 1077; 2024-11-12T10:56:20,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731408980825, max timestamp=1731408980842Num uniques 1077; 2024-11-12T10:56:20,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T10:56:20,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T10:56:20,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1731408980874, max timestamp=1731408980908Num uniques 1086; 2024-11-12T10:56:20,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49726, new segment data size=8506, suffix heap size=140912,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T10:56:20,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1731408980874, max timestamp=1731408980908Num uniques 1086; 2024-11-12T10:56:20,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8515, totalHeapSize=19584, min timestamp=1731408980874, max timestamp=1731408980908Num uniques 1086; 2024-11-12T10:56:20,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49726, new segment data size=8506, suffix heap size=140912,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T10:56:20,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49735, new segment data size=8515, suffix heap size=140912,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T10:56:20,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731408980948, max timestamp=1731408980961Num uniques 1045; 2024-11-12T10:56:20,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-12T10:56:20,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731408980948, max timestamp=1731408980961Num uniques 1045; 2024-11-12T10:56:20,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731408980948, max timestamp=1731408980961Num uniques 1045; 2024-11-12T10:56:20,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-12T10:56:20,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-12T10:56:21,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409041011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:21,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409041014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:21,045 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/f62ee5777c7647bca4d2312cbd8fe1bd as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f62ee5777c7647bca4d2312cbd8fe1bd 2024-11-12T10:56:21,048 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c83771492407a48a02121732c6276538/C of c83771492407a48a02121732c6276538 into f62ee5777c7647bca4d2312cbd8fe1bd(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:21,048 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:21,048 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/C, priority=12, startTime=1731408980201; duration=0sec 2024-11-12T10:56:21,048 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:21,048 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:C 2024-11-12T10:56:21,120 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/bc24a8cc674244938f1a7582bc2a64ca 2024-11-12T10:56:21,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/452182ddecd744b6ba211756ac3a61d8 is 50, key is test_row_0/C:col10/1731408978561/Put/seqid=0 2024-11-12T10:56:21,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742376_1552 (size=12151) 2024-11-12T10:56:21,129 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/452182ddecd744b6ba211756ac3a61d8 2024-11-12T10:56:21,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409041127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/539765ba6cc3427499abd8f7e4811675 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/539765ba6cc3427499abd8f7e4811675 2024-11-12T10:56:21,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409041127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:21,134 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/539765ba6cc3427499abd8f7e4811675, entries=150, sequenceid=243, filesize=30.4 K 2024-11-12T10:56:21,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/bc24a8cc674244938f1a7582bc2a64ca as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/bc24a8cc674244938f1a7582bc2a64ca 2024-11-12T10:56:21,137 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/bc24a8cc674244938f1a7582bc2a64ca, entries=150, sequenceid=243, filesize=11.9 K 2024-11-12T10:56:21,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/452182ddecd744b6ba211756ac3a61d8 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/452182ddecd744b6ba211756ac3a61d8 2024-11-12T10:56:21,140 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/452182ddecd744b6ba211756ac3a61d8, entries=150, sequenceid=243, filesize=11.9 K 2024-11-12T10:56:21,141 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for c83771492407a48a02121732c6276538 in 844ms, sequenceid=243, compaction requested=false 2024-11-12T10:56:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-12T10:56:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-12T10:56:21,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-12T10:56:21,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5220 sec 2024-11-12T10:56:21,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.5260 sec 2024-11-12T10:56:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on c83771492407a48a02121732c6276538 2024-11-12T10:56:21,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=154.31 KB heapSize=437.39 KB 2024-11-12T10:56:21,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:21,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T10:56:21,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:21,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T10:56:21,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:21,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T10:56:21,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111202548d0e847d44b7a0a6c999d0360ed3_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408980997/Put/seqid=0 2024-11-12T10:56:21,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742377_1553 (size=12454) 2024-11-12T10:56:21,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409041346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:21,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409041351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:21,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409041457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:21,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:21,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409041467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:21,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409041667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:21,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409041676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:21,749 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:21,752 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111202548d0e847d44b7a0a6c999d0360ed3_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111202548d0e847d44b7a0a6c999d0360ed3_c83771492407a48a02121732c6276538 2024-11-12T10:56:21,753 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/70db3aeff4d040419bb2dc41c56bc172, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:21,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/70db3aeff4d040419bb2dc41c56bc172 is 175, key is test_row_0/A:col10/1731408980997/Put/seqid=0 2024-11-12T10:56:21,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742378_1554 (size=31255) 2024-11-12T10:56:21,756 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/70db3aeff4d040419bb2dc41c56bc172 2024-11-12T10:56:21,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/adc296be250b4eb7bc4cd5d6c3734ed6 is 50, key is test_row_0/B:col10/1731408980997/Put/seqid=0 2024-11-12T10:56:21,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742379_1555 (size=12301) 2024-11-12T10:56:21,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409041807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:21,811 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:21,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409041977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:21,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:21,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409041987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:22,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/adc296be250b4eb7bc4cd5d6c3734ed6 2024-11-12T10:56:22,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/12d40dbd59fa456b9321a69365e79838 is 50, key is test_row_0/C:col10/1731408980997/Put/seqid=0 2024-11-12T10:56:22,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742380_1556 (size=12301) 2024-11-12T10:56:22,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409042487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:22,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409042497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:22,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:22,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:22,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:22,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/12d40dbd59fa456b9321a69365e79838 2024-11-12T10:56:22,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/70db3aeff4d040419bb2dc41c56bc172 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/70db3aeff4d040419bb2dc41c56bc172 2024-11-12T10:56:22,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/70db3aeff4d040419bb2dc41c56bc172, entries=150, sequenceid=353, filesize=30.5 K 2024-11-12T10:56:22,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/adc296be250b4eb7bc4cd5d6c3734ed6 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/adc296be250b4eb7bc4cd5d6c3734ed6 2024-11-12T10:56:22,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/adc296be250b4eb7bc4cd5d6c3734ed6, entries=150, sequenceid=353, filesize=12.0 K 2024-11-12T10:56:22,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/12d40dbd59fa456b9321a69365e79838 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/12d40dbd59fa456b9321a69365e79838 2024-11-12T10:56:22,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/12d40dbd59fa456b9321a69365e79838, entries=150, sequenceid=353, filesize=12.0 K 2024-11-12T10:56:22,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~437.34 KB/447840, currentSize=26.84 KB/27480 for c83771492407a48a02121732c6276538 in 1250ms, sequenceid=353, compaction requested=true 2024-11-12T10:56:22,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:22,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:56:22,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:22,589 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:22,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:56:22,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:22,589 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:22,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:56:22,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:22,589 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:22,589 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:22,589 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/B is initiating minor compaction (all files) 2024-11-12T10:56:22,589 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/A is initiating minor compaction (all files) 2024-11-12T10:56:22,589 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/B in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:22,589 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/A in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:22,589 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b6cfbffb1372481ca99eb4902e44bde2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/539765ba6cc3427499abd8f7e4811675, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/70db3aeff4d040419bb2dc41c56bc172] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=91.5 K 2024-11-12T10:56:22,589 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/f818c9de68c246a194e39d5d2df8744b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/bc24a8cc674244938f1a7582bc2a64ca, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/adc296be250b4eb7bc4cd5d6c3734ed6] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=36.0 K 2024-11-12T10:56:22,590 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:22,590 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b6cfbffb1372481ca99eb4902e44bde2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/539765ba6cc3427499abd8f7e4811675, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/70db3aeff4d040419bb2dc41c56bc172] 2024-11-12T10:56:22,590 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting f818c9de68c246a194e39d5d2df8744b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731408977586 2024-11-12T10:56:22,590 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6cfbffb1372481ca99eb4902e44bde2, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731408977586 2024-11-12T10:56:22,590 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting bc24a8cc674244938f1a7582bc2a64ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731408978561 2024-11-12T10:56:22,590 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 539765ba6cc3427499abd8f7e4811675, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731408978561 2024-11-12T10:56:22,591 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting adc296be250b4eb7bc4cd5d6c3734ed6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731408980997 2024-11-12T10:56:22,591 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70db3aeff4d040419bb2dc41c56bc172, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731408980997 2024-11-12T10:56:22,595 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:22,596 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#B#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:22,596 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/dd9b3451c51e478d91d5231ce29fc7e1 is 50, key is test_row_0/B:col10/1731408980997/Put/seqid=0 2024-11-12T10:56:22,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742381_1557 (size=12643) 2024-11-12T10:56:22,600 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112ad7d8c471cf74ca48f07bb7b10831111_c83771492407a48a02121732c6276538 store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:22,602 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112ad7d8c471cf74ca48f07bb7b10831111_c83771492407a48a02121732c6276538, store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:22,602 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ad7d8c471cf74ca48f07bb7b10831111_c83771492407a48a02121732c6276538 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:22,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742382_1558 (size=4469) 2024-11-12T10:56:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T10:56:22,777 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:56:22,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:56:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-12T10:56:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T10:56:22,779 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:56:22,779 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:56:22,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:56:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T10:56:22,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T10:56:22,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:22,931 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T10:56:22,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:22,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:22,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:22,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:22,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:22,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:22,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112850eb8e4127b4b13a087920026628363_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408981338/Put/seqid=0 2024-11-12T10:56:22,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742383_1559 (size=12454) 2024-11-12T10:56:23,003 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/dd9b3451c51e478d91d5231ce29fc7e1 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/dd9b3451c51e478d91d5231ce29fc7e1 2024-11-12T10:56:23,011 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#A#compaction#476 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:23,012 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/042c0adfabfe4e1c9bf89c6608d62303 is 175, key is test_row_0/A:col10/1731408980997/Put/seqid=0 2024-11-12T10:56:23,014 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/B of c83771492407a48a02121732c6276538 into dd9b3451c51e478d91d5231ce29fc7e1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:23,014 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:23,014 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/B, priority=13, startTime=1731408982589; duration=0sec 2024-11-12T10:56:23,014 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:23,014 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:B 2024-11-12T10:56:23,014 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:23,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742384_1560 (size=31597) 2024-11-12T10:56:23,015 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:23,015 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/C is initiating minor compaction (all files) 2024-11-12T10:56:23,015 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/C in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:23,015 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f62ee5777c7647bca4d2312cbd8fe1bd, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/452182ddecd744b6ba211756ac3a61d8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/12d40dbd59fa456b9321a69365e79838] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=36.0 K 2024-11-12T10:56:23,016 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting f62ee5777c7647bca4d2312cbd8fe1bd, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731408977586 2024-11-12T10:56:23,016 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 452182ddecd744b6ba211756ac3a61d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731408978561 2024-11-12T10:56:23,016 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 12d40dbd59fa456b9321a69365e79838, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731408980997 2024-11-12T10:56:23,019 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/042c0adfabfe4e1c9bf89c6608d62303 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/042c0adfabfe4e1c9bf89c6608d62303 2024-11-12T10:56:23,023 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#C#compaction#479 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:23,023 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/fe15e0a4ce09491c817ff9089707c118 is 50, key is test_row_0/C:col10/1731408980997/Put/seqid=0 2024-11-12T10:56:23,025 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/A of c83771492407a48a02121732c6276538 into 042c0adfabfe4e1c9bf89c6608d62303(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:23,025 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:23,025 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/A, priority=13, startTime=1731408982588; duration=0sec 2024-11-12T10:56:23,025 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:23,025 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:A 2024-11-12T10:56:23,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742385_1561 (size=12643) 2024-11-12T10:56:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T10:56:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:23,344 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112850eb8e4127b4b13a087920026628363_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112850eb8e4127b4b13a087920026628363_c83771492407a48a02121732c6276538 2024-11-12T10:56:23,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/706a751267e844859aa1f6f470091566, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:23,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/706a751267e844859aa1f6f470091566 is 175, key is test_row_0/A:col10/1731408981338/Put/seqid=0 2024-11-12T10:56:23,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742386_1562 (size=31255) 2024-11-12T10:56:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T10:56:23,430 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/fe15e0a4ce09491c817ff9089707c118 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/fe15e0a4ce09491c817ff9089707c118 2024-11-12T10:56:23,433 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/C of c83771492407a48a02121732c6276538 into fe15e0a4ce09491c817ff9089707c118(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:23,433 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:23,433 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/C, priority=13, startTime=1731408982589; duration=0sec 2024-11-12T10:56:23,433 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:23,433 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:C 2024-11-12T10:56:23,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on c83771492407a48a02121732c6276538 2024-11-12T10:56:23,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731408983523, max timestamp=1731408983533Num uniques 859; 2024-11-12T10:56:23,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731408983523, max timestamp=1731408983533Num uniques 859; 2024-11-12T10:56:23,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T10:56:23,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T10:56:23,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731408983523, max timestamp=1731408983533Num uniques 859; 2024-11-12T10:56:23,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T10:56:23,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731408983569, max timestamp=1731408983584Num uniques 1068; 2024-11-12T10:56:23,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T10:56:23,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731408983569, max timestamp=1731408983584Num uniques 1068; 2024-11-12T10:56:23,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731408983569, max timestamp=1731408983584Num uniques 1068; 2024-11-12T10:56:23,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T10:56:23,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T10:56:23,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731408983613, max timestamp=1731408983626Num uniques 1027; 2024-11-12T10:56:23,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-12T10:56:23,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731408983613, max timestamp=1731408983626Num uniques 1027; 2024-11-12T10:56:23,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731408983613, max timestamp=1731408983626Num uniques 1027; 2024-11-12T10:56:23,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-12T10:56:23,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-12T10:56:23,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731408983657, max timestamp=1731408983671Num uniques 1086; 2024-11-12T10:56:23,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T10:56:23,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731408983657, max timestamp=1731408983671Num uniques 1086; 2024-11-12T10:56:23,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-12T10:56:23,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731408983657, max timestamp=1731408983671Num uniques 1086; 2024-11-12T10:56:23,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T10:56:23,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731408983703, max timestamp=1731408983716Num uniques 1095; 2024-11-12T10:56:23,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731408983703, max timestamp=1731408983716Num uniques 1045; 2024-11-12T10:56:23,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-12T10:56:23,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-12T10:56:23,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731408983703, max timestamp=1731408983716Num uniques 1045; 2024-11-12T10:56:23,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-12T10:56:23,751 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/706a751267e844859aa1f6f470091566 2024-11-12T10:56:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/0e6bb15814ab4d7ea4314b42511a85b5 is 50, key is test_row_0/B:col10/1731408981338/Put/seqid=0 2024-11-12T10:56:23,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742387_1563 (size=12301) 2024-11-12T10:56:23,759 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/0e6bb15814ab4d7ea4314b42511a85b5 2024-11-12T10:56:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/f7eb42babe9a4214a390297e6cab4298 is 50, key is test_row_0/C:col10/1731408981338/Put/seqid=0 2024-11-12T10:56:23,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731408983742, max timestamp=1731408983763Num uniques 1104; 2024-11-12T10:56:23,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-12T10:56:23,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731408983742, max timestamp=1731408983763Num uniques 1104; 2024-11-12T10:56:23,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-12T10:56:23,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731408983756, max timestamp=1731408983763Num uniques 1104; 2024-11-12T10:56:23,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=6870, suffix heap size=142784,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=150 2024-11-12T10:56:23,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742388_1564 (size=12301) 2024-11-12T10:56:23,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731408983802, max timestamp=1731408983812Num uniques 1063; 2024-11-12T10:56:23,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392  suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-12T10:56:23,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731408983802, max timestamp=1731408983812Num uniques 1063; 2024-11-12T10:56:23,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392  suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-12T10:56:23,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731408983802, max timestamp=1731408983812Num uniques 1009; 2024-11-12T10:56:23,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46208, new segment data size=9748, suffix heap size=132904,new segment heap size=22392  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=213 2024-11-12T10:56:23,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731408983844, max timestamp=1731408983858Num uniques 1072; 2024-11-12T10:56:23,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731408983844, max timestamp=1731408983858Num uniques 1072; 2024-11-12T10:56:23,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328  suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-12T10:56:23,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328  suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-12T10:56:23,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731408983844, max timestamp=1731408983858Num uniques 1072; 2024-11-12T10:56:23,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328  suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-12T10:56:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T10:56:23,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731408983877, max timestamp=1731408983898Num uniques 1081; 2024-11-12T10:56:23,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264  suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-12T10:56:23,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731408983877, max timestamp=1731408983898Num uniques 1081; 2024-11-12T10:56:23,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264  suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-12T10:56:23,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731408983877, max timestamp=1731408983898Num uniques 1081; 2024-11-12T10:56:23,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264  suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-12T10:56:23,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731408983950, max timestamp=1731408983959Num uniques 1090; 2024-11-12T10:56:23,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000  suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-12T10:56:23,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731408983950, max timestamp=1731408983959Num uniques 1090; 2024-11-12T10:56:23,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000  suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-12T10:56:23,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731408983950, max timestamp=1731408983959Num uniques 1090; 2024-11-12T10:56:23,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000  suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-12T10:56:24,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731408983972, max timestamp=1731408984001Num uniques 1049; 2024-11-12T10:56:24,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936  suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-12T10:56:24,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731408983972, max timestamp=1731408984001Num uniques 1049; 2024-11-12T10:56:24,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936  suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-12T10:56:24,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731408983972, max timestamp=1731408984001Num uniques 1049; 2024-11-12T10:56:24,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936  suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-12T10:56:24,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731408984037, max timestamp=1731408984047Num uniques 1058; 2024-11-12T10:56:24,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731408984037, max timestamp=1731408984047Num uniques 1058; 2024-11-12T10:56:24,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731408984037, max timestamp=1731408984047Num uniques 1058; 2024-11-12T10:56:24,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672  suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-12T10:56:24,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672  suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-12T10:56:24,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672  suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-12T10:56:24,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731408984061, max timestamp=1731408984087Num uniques 1017; 2024-11-12T10:56:24,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731408984061, max timestamp=1731408984087Num uniques 1017; 2024-11-12T10:56:24,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808  suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-12T10:56:24,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808  suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-12T10:56:24,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731408984061, max timestamp=1731408984087Num uniques 1017; 2024-11-12T10:56:24,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808  suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-12T10:56:24,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8047, totalHeapSize=18544, min timestamp=1731408984120, max timestamp=1731408984134Num uniques 1076; 2024-11-12T10:56:24,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731408984120, max timestamp=1731408984134Num uniques 1076; 2024-11-12T10:56:24,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731408984120, max timestamp=1731408984134Num uniques 1076; 2024-11-12T10:56:24,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544  suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-11-12T10:56:24,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49267, new segment data size=8047, suffix heap size=139872,new segment heap size=18544  suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-11-12T10:56:24,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aed202db7571:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544  suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-11-12T10:56:24,176 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/f7eb42babe9a4214a390297e6cab4298 2024-11-12T10:56:24,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/706a751267e844859aa1f6f470091566 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/706a751267e844859aa1f6f470091566 2024-11-12T10:56:24,183 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/706a751267e844859aa1f6f470091566, entries=150, sequenceid=360, filesize=30.5 K 2024-11-12T10:56:24,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/0e6bb15814ab4d7ea4314b42511a85b5 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/0e6bb15814ab4d7ea4314b42511a85b5 2024-11-12T10:56:24,188 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/0e6bb15814ab4d7ea4314b42511a85b5, entries=150, sequenceid=360, filesize=12.0 K 2024-11-12T10:56:24,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/f7eb42babe9a4214a390297e6cab4298 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f7eb42babe9a4214a390297e6cab4298 2024-11-12T10:56:24,191 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f7eb42babe9a4214a390297e6cab4298, entries=150, sequenceid=360, filesize=12.0 K 2024-11-12T10:56:24,211 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=87.22 KB/89310 for c83771492407a48a02121732c6276538 in 1279ms, sequenceid=360, compaction requested=false 2024-11-12T10:56:24,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:24,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:24,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-12T10:56:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on c83771492407a48a02121732c6276538 2024-11-12T10:56:24,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=100.63 KB heapSize=283.78 KB 2024-11-12T10:56:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:56:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:56:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:24,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T10:56:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-12T10:56:24,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-12T10:56:24,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4340 sec 2024-11-12T10:56:24,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.4430 sec 2024-11-12T10:56:24,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e21f25f40ffc47abb1a6ff767f2a45b2_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408984174/Put/seqid=0 2024-11-12T10:56:24,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742389_1565 (size=17534) 2024-11-12T10:56:24,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409044271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:24,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:24,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409044276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:24,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409044387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:24,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409044387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:24,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409044597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:24,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409044597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:24,634 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:24,636 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e21f25f40ffc47abb1a6ff767f2a45b2_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e21f25f40ffc47abb1a6ff767f2a45b2_c83771492407a48a02121732c6276538 2024-11-12T10:56:24,636 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/05e491cf04d149e398db82e99a4ef259, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:24,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/05e491cf04d149e398db82e99a4ef259 is 175, key is test_row_0/A:col10/1731408984174/Put/seqid=0 2024-11-12T10:56:24,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742390_1566 (size=48639) 2024-11-12T10:56:24,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409044906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409044907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T10:56:24,927 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:56:24,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:56:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-12T10:56:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T10:56:24,929 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:56:24,929 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:56:24,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:56:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T10:56:25,040 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=618, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/05e491cf04d149e398db82e99a4ef259 2024-11-12T10:56:25,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/2885915e3a4a49ff876f67582118070f is 50, key is test_row_0/B:col10/1731408984174/Put/seqid=0 2024-11-12T10:56:25,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742391_1567 (size=12301) 2024-11-12T10:56:25,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T10:56:25,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:25,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T10:56:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T10:56:25,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T10:56:25,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:25,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409045416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:25,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409045417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:25,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/2885915e3a4a49ff876f67582118070f 2024-11-12T10:56:25,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/7cec1232476f458690f977f83018d0b3 is 50, key is test_row_0/C:col10/1731408984174/Put/seqid=0 2024-11-12T10:56:25,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742392_1568 (size=12301) 2024-11-12T10:56:25,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T10:56:25,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:25,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T10:56:25,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T10:56:25,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:25,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T10:56:25,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:25,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:25,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37546 deadline: 1731409045847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:25,851 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:25,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/7cec1232476f458690f977f83018d0b3 2024-11-12T10:56:25,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/05e491cf04d149e398db82e99a4ef259 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/05e491cf04d149e398db82e99a4ef259 2024-11-12T10:56:25,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/05e491cf04d149e398db82e99a4ef259, entries=250, sequenceid=618, filesize=47.5 K 2024-11-12T10:56:25,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/2885915e3a4a49ff876f67582118070f as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/2885915e3a4a49ff876f67582118070f 2024-11-12T10:56:25,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/2885915e3a4a49ff876f67582118070f, entries=150, sequenceid=618, filesize=12.0 K 2024-11-12T10:56:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/7cec1232476f458690f977f83018d0b3 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/7cec1232476f458690f977f83018d0b3 2024-11-12T10:56:25,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/7cec1232476f458690f977f83018d0b3, entries=150, sequenceid=618, filesize=12.0 K 2024-11-12T10:56:25,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~283.73 KB/290544, currentSize=80.51 KB/82440 for c83771492407a48a02121732c6276538 in 1661ms, sequenceid=618, compaction requested=true 2024-11-12T10:56:25,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:25,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:56:25,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:25,872 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:25,872 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:25,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:56:25,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:25,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:56:25,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:25,873 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111491 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:25,873 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/A is initiating minor compaction (all files) 2024-11-12T10:56:25,873 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/A in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,873 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/042c0adfabfe4e1c9bf89c6608d62303, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/706a751267e844859aa1f6f470091566, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/05e491cf04d149e398db82e99a4ef259] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=108.9 K 2024-11-12T10:56:25,873 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,873 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/042c0adfabfe4e1c9bf89c6608d62303, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/706a751267e844859aa1f6f470091566, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/05e491cf04d149e398db82e99a4ef259] 2024-11-12T10:56:25,873 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 042c0adfabfe4e1c9bf89c6608d62303, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731408980997 2024-11-12T10:56:25,874 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:25,874 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/B is initiating minor compaction (all files) 2024-11-12T10:56:25,874 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/B in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,874 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 706a751267e844859aa1f6f470091566, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731408981338 2024-11-12T10:56:25,874 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/dd9b3451c51e478d91d5231ce29fc7e1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/0e6bb15814ab4d7ea4314b42511a85b5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/2885915e3a4a49ff876f67582118070f] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=36.4 K 2024-11-12T10:56:25,874 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05e491cf04d149e398db82e99a4ef259, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731408984170 2024-11-12T10:56:25,874 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting dd9b3451c51e478d91d5231ce29fc7e1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731408980997 2024-11-12T10:56:25,874 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e6bb15814ab4d7ea4314b42511a85b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731408981338 2024-11-12T10:56:25,875 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 2885915e3a4a49ff876f67582118070f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731408984170 2024-11-12T10:56:25,878 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:25,879 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411124e5d3adcebd74ef2bac422e26cc579a6_c83771492407a48a02121732c6276538 store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:25,880 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#B#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:25,880 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/632bfd0dc5b64696841831027cab192c is 50, key is test_row_0/B:col10/1731408984174/Put/seqid=0 2024-11-12T10:56:25,881 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411124e5d3adcebd74ef2bac422e26cc579a6_c83771492407a48a02121732c6276538, store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:25,881 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124e5d3adcebd74ef2bac422e26cc579a6_c83771492407a48a02121732c6276538 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:25,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742393_1569 (size=12745) 2024-11-12T10:56:25,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742394_1570 (size=4469) 2024-11-12T10:56:25,887 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#A#compaction#485 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:25,887 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/24b6d99d01b44b54b25b7b6419ee0efc is 175, key is test_row_0/A:col10/1731408984174/Put/seqid=0 2024-11-12T10:56:25,889 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/632bfd0dc5b64696841831027cab192c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/632bfd0dc5b64696841831027cab192c 2024-11-12T10:56:25,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742395_1571 (size=31699) 2024-11-12T10:56:25,893 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/B of c83771492407a48a02121732c6276538 into 632bfd0dc5b64696841831027cab192c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:25,893 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:25,893 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/B, priority=13, startTime=1731408985872; duration=0sec 2024-11-12T10:56:25,893 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:25,893 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:B 2024-11-12T10:56:25,894 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:25,894 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:25,894 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/C is initiating minor compaction (all files) 2024-11-12T10:56:25,894 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/C in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,894 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/fe15e0a4ce09491c817ff9089707c118, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f7eb42babe9a4214a390297e6cab4298, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/7cec1232476f458690f977f83018d0b3] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=36.4 K 2024-11-12T10:56:25,895 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting fe15e0a4ce09491c817ff9089707c118, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731408980997 2024-11-12T10:56:25,895 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting f7eb42babe9a4214a390297e6cab4298, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731408981338 2024-11-12T10:56:25,895 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cec1232476f458690f977f83018d0b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731408984170 2024-11-12T10:56:25,902 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#C#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:25,903 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/c6fdbae10de84477b37e19882d9eb9ab is 50, key is test_row_0/C:col10/1731408984174/Put/seqid=0 2024-11-12T10:56:25,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742396_1572 (size=12745) 2024-11-12T10:56:25,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T10:56:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:25,997 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T10:56:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:26,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f2ca60a81fd6434c9a5b3f33fc5292ac_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408984263/Put/seqid=0 2024-11-12T10:56:26,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742397_1573 (size=12454) 2024-11-12T10:56:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T10:56:26,295 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/24b6d99d01b44b54b25b7b6419ee0efc as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/24b6d99d01b44b54b25b7b6419ee0efc 2024-11-12T10:56:26,298 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/A of c83771492407a48a02121732c6276538 into 24b6d99d01b44b54b25b7b6419ee0efc(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:26,298 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:26,298 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/A, priority=13, startTime=1731408985872; duration=0sec 2024-11-12T10:56:26,298 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:26,298 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:A 2024-11-12T10:56:26,309 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/c6fdbae10de84477b37e19882d9eb9ab as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/c6fdbae10de84477b37e19882d9eb9ab 2024-11-12T10:56:26,312 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/C of c83771492407a48a02121732c6276538 into c6fdbae10de84477b37e19882d9eb9ab(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:26,312 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:26,312 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/C, priority=13, startTime=1731408985872; duration=0sec 2024-11-12T10:56:26,312 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:26,312 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:C 2024-11-12T10:56:26,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:26,407 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f2ca60a81fd6434c9a5b3f33fc5292ac_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f2ca60a81fd6434c9a5b3f33fc5292ac_c83771492407a48a02121732c6276538 2024-11-12T10:56:26,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/1fb39cbaca6a402bac359d9dea4f5930, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:26,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/1fb39cbaca6a402bac359d9dea4f5930 is 175, key is test_row_0/A:col10/1731408984263/Put/seqid=0 2024-11-12T10:56:26,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742398_1574 (size=31255) 2024-11-12T10:56:26,411 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=634, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/1fb39cbaca6a402bac359d9dea4f5930 2024-11-12T10:56:26,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/91c9eed56dd642aabaa168583af0e365 is 50, key is test_row_0/B:col10/1731408984263/Put/seqid=0 2024-11-12T10:56:26,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742399_1575 (size=12301) 2024-11-12T10:56:26,419 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/91c9eed56dd642aabaa168583af0e365 2024-11-12T10:56:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/c8df7701efe14f839084bfd47349345d is 50, key is test_row_0/C:col10/1731408984263/Put/seqid=0 2024-11-12T10:56:26,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742400_1576 (size=12301) 2024-11-12T10:56:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on c83771492407a48a02121732c6276538 2024-11-12T10:56:26,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:26,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409046456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:26,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409046461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:26,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:26,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:26,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:26,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409046566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:26,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409046567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:26,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409046777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:26,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409046777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:26,827 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/c8df7701efe14f839084bfd47349345d 2024-11-12T10:56:26,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/1fb39cbaca6a402bac359d9dea4f5930 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/1fb39cbaca6a402bac359d9dea4f5930 2024-11-12T10:56:26,833 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/1fb39cbaca6a402bac359d9dea4f5930, entries=150, sequenceid=634, filesize=30.5 K 2024-11-12T10:56:26,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/91c9eed56dd642aabaa168583af0e365 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/91c9eed56dd642aabaa168583af0e365 2024-11-12T10:56:26,836 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/91c9eed56dd642aabaa168583af0e365, entries=150, sequenceid=634, filesize=12.0 K 2024-11-12T10:56:26,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/c8df7701efe14f839084bfd47349345d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/c8df7701efe14f839084bfd47349345d 2024-11-12T10:56:26,840 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/c8df7701efe14f839084bfd47349345d, entries=150, sequenceid=634, filesize=12.0 K 2024-11-12T10:56:26,840 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c83771492407a48a02121732c6276538 in 843ms, sequenceid=634, compaction requested=false 2024-11-12T10:56:26,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:26,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:26,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-12T10:56:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-12T10:56:26,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-12T10:56:26,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9130 sec 2024-11-12T10:56:26,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.9150 sec 2024-11-12T10:56:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T10:56:27,077 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:56:27,078 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T10:56:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-12T10:56:27,079 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T10:56:27,080 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T10:56:27,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T10:56:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T10:56:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on c83771492407a48a02121732c6276538 2024-11-12T10:56:27,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T10:56:27,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:27,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:27,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:27,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:27,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:27,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:27,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112dbdd30ce874949119ad5ee188aef8730_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408986459/Put/seqid=0 2024-11-12T10:56:27,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742401_1577 (size=14994) 2024-11-12T10:56:27,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409047113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409047114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T10:56:27,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409047226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409047227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:27,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-12T10:56:27,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:27,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:27,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:27,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-12T10:56:27,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:27,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:27,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:27,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T10:56:27,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409047436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:27,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409047436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:27,498 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:27,501 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112dbdd30ce874949119ad5ee188aef8730_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112dbdd30ce874949119ad5ee188aef8730_c83771492407a48a02121732c6276538 2024-11-12T10:56:27,502 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/c68a1191ffe648f39a9e5c5bc98773dd, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:27,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/c68a1191ffe648f39a9e5c5bc98773dd is 175, key is test_row_0/A:col10/1731408986459/Put/seqid=0 2024-11-12T10:56:27,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742402_1578 (size=39949) 2024-11-12T10:56:27,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-12T10:56:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:27,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-12T10:56:27,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:27,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:27,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:27,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T10:56:27,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409047747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:27,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:27,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409047747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:27,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-12T10:56:27,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:27,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:27,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:27,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,911 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=654, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/c68a1191ffe648f39a9e5c5bc98773dd 2024-11-12T10:56:27,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/97b02bacb27d4fff96b400668493e31e is 50, key is test_row_0/B:col10/1731408986459/Put/seqid=0 2024-11-12T10:56:27,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742403_1579 (size=12301) 2024-11-12T10:56:27,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-12T10:56:27,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:27,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:27,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:27,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:28,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-12T10:56:28,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:28,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:28,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:28,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:28,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T10:56:28,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409048257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:28,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:28,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:28,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:28,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409048257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:28,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:28,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:28,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:28,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-12T10:56:28,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:28,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:28,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:28,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:28,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:28,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=654 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/97b02bacb27d4fff96b400668493e31e 2024-11-12T10:56:28,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/2899be625eab49138fe5b59d3a27b1e9 is 50, key is test_row_0/C:col10/1731408986459/Put/seqid=0 2024-11-12T10:56:28,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742404_1580 (size=12301) 2024-11-12T10:56:28,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-12T10:56:28,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:28,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:28,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:28,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:28,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:28,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-12T10:56:28,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:28,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:28,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:28,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:28,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:56:28,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37608 deadline: 1731409048627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:28,629 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_0' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:28,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37572 deadline: 1731409048657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:28,660 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to aed202db7571,43143,1731408812834 for 'test_row_2' in c83771492407a48a02121732c6276538 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18294 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T10:56:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:28,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=654 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/2899be625eab49138fe5b59d3a27b1e9 2024-11-12T10:56:28,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/c68a1191ffe648f39a9e5c5bc98773dd as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/c68a1191ffe648f39a9e5c5bc98773dd 2024-11-12T10:56:28,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/c68a1191ffe648f39a9e5c5bc98773dd, entries=200, sequenceid=654, filesize=39.0 K 2024-11-12T10:56:28,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/97b02bacb27d4fff96b400668493e31e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/97b02bacb27d4fff96b400668493e31e 2024-11-12T10:56:28,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/97b02bacb27d4fff96b400668493e31e, entries=150, sequenceid=654, filesize=12.0 K 2024-11-12T10:56:28,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/2899be625eab49138fe5b59d3a27b1e9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/2899be625eab49138fe5b59d3a27b1e9 2024-11-12T10:56:28,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/2899be625eab49138fe5b59d3a27b1e9, entries=150, sequenceid=654, filesize=12.0 K 2024-11-12T10:56:28,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c83771492407a48a02121732c6276538 in 1651ms, sequenceid=654, compaction requested=true 2024-11-12T10:56:28,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:28,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:56:28,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:28,740 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:28,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:56:28,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:28,740 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:28,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:56:28,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:28,741 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:28,741 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:28,741 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/B is initiating minor compaction (all files) 2024-11-12T10:56:28,741 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/A is initiating minor compaction (all files) 2024-11-12T10:56:28,741 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/B in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:28,741 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/A in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:28,741 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/24b6d99d01b44b54b25b7b6419ee0efc, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/1fb39cbaca6a402bac359d9dea4f5930, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/c68a1191ffe648f39a9e5c5bc98773dd] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=100.5 K 2024-11-12T10:56:28,741 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/632bfd0dc5b64696841831027cab192c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/91c9eed56dd642aabaa168583af0e365, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/97b02bacb27d4fff96b400668493e31e] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=36.5 K 2024-11-12T10:56:28,741 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:28,741 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/24b6d99d01b44b54b25b7b6419ee0efc, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/1fb39cbaca6a402bac359d9dea4f5930, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/c68a1191ffe648f39a9e5c5bc98773dd] 2024-11-12T10:56:28,742 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24b6d99d01b44b54b25b7b6419ee0efc, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731408984170 2024-11-12T10:56:28,742 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 632bfd0dc5b64696841831027cab192c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731408984170 2024-11-12T10:56:28,742 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fb39cbaca6a402bac359d9dea4f5930, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1731408984225 2024-11-12T10:56:28,742 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 91c9eed56dd642aabaa168583af0e365, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1731408984225 2024-11-12T10:56:28,742 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting c68a1191ffe648f39a9e5c5bc98773dd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1731408986443 2024-11-12T10:56:28,742 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 97b02bacb27d4fff96b400668493e31e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1731408986443 2024-11-12T10:56:28,749 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:28,750 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#B#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:28,750 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/d801ef95452c4e399b4d0f060d8ab09c is 50, key is test_row_0/B:col10/1731408986459/Put/seqid=0 2024-11-12T10:56:28,751 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112e7732d204c7d40b787c74ff2bf3db687_c83771492407a48a02121732c6276538 store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:28,753 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112e7732d204c7d40b787c74ff2bf3db687_c83771492407a48a02121732c6276538, store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:28,753 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e7732d204c7d40b787c74ff2bf3db687_c83771492407a48a02121732c6276538 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:28,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43143 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-12T10:56:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:28,757 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T10:56:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:28,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742406_1582 (size=4469) 2024-11-12T10:56:28,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f4f4dfc1a75e41488ffcfbb6e2845149_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408987113/Put/seqid=0 2024-11-12T10:56:28,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742405_1581 (size=12847) 2024-11-12T10:56:28,779 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/d801ef95452c4e399b4d0f060d8ab09c as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/d801ef95452c4e399b4d0f060d8ab09c 2024-11-12T10:56:28,785 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/B of c83771492407a48a02121732c6276538 into d801ef95452c4e399b4d0f060d8ab09c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:28,785 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:28,785 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/B, priority=13, startTime=1731408988740; duration=0sec 2024-11-12T10:56:28,785 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:28,785 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:B 2024-11-12T10:56:28,785 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:28,786 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:28,786 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/C is initiating minor compaction (all files) 2024-11-12T10:56:28,786 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/C in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:28,786 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/c6fdbae10de84477b37e19882d9eb9ab, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/c8df7701efe14f839084bfd47349345d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/2899be625eab49138fe5b59d3a27b1e9] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=36.5 K 2024-11-12T10:56:28,787 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting c6fdbae10de84477b37e19882d9eb9ab, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731408984170 2024-11-12T10:56:28,787 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting c8df7701efe14f839084bfd47349345d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1731408984225 2024-11-12T10:56:28,787 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 2899be625eab49138fe5b59d3a27b1e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1731408986443 2024-11-12T10:56:28,797 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#C#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:28,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742407_1583 (size=12454) 2024-11-12T10:56:28,798 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/deddd2cc35aa4a67a35a340a94c57fd6 is 50, key is test_row_0/C:col10/1731408986459/Put/seqid=0 2024-11-12T10:56:28,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742408_1584 (size=12847) 2024-11-12T10:56:29,167 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#A#compaction#495 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:29,167 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/f7561a68f168439b98543af4c3e93edb is 175, key is test_row_0/A:col10/1731408986459/Put/seqid=0 2024-11-12T10:56:29,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742409_1585 (size=31801) 2024-11-12T10:56:29,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:29,200 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f4f4dfc1a75e41488ffcfbb6e2845149_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f4f4dfc1a75e41488ffcfbb6e2845149_c83771492407a48a02121732c6276538 2024-11-12T10:56:29,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/b0405b74c69b4708ac3a6472e3df8bd3, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:29,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/b0405b74c69b4708ac3a6472e3df8bd3 is 175, key is test_row_0/A:col10/1731408987113/Put/seqid=0 2024-11-12T10:56:29,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742410_1586 (size=31255) 2024-11-12T10:56:29,212 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/deddd2cc35aa4a67a35a340a94c57fd6 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/deddd2cc35aa4a67a35a340a94c57fd6 2024-11-12T10:56:29,215 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/C of c83771492407a48a02121732c6276538 into deddd2cc35aa4a67a35a340a94c57fd6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:29,215 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:29,215 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/C, priority=13, startTime=1731408988740; duration=0sec 2024-11-12T10:56:29,215 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:29,215 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:C 2024-11-12T10:56:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T10:56:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on c83771492407a48a02121732c6276538 2024-11-12T10:56:29,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. as already flushing 2024-11-12T10:56:29,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409049324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:29,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:29,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409049326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:29,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409049437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:29,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:29,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409049437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:29,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:29,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:29,574 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/f7561a68f168439b98543af4c3e93edb as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/f7561a68f168439b98543af4c3e93edb 2024-11-12T10:56:29,577 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/A of c83771492407a48a02121732c6276538 into f7561a68f168439b98543af4c3e93edb(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:29,577 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:29,577 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/A, priority=13, startTime=1731408988740; duration=0sec 2024-11-12T10:56:29,577 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:29,577 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:A 2024-11-12T10:56:29,605 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=668, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/b0405b74c69b4708ac3a6472e3df8bd3 2024-11-12T10:56:29,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/3ae1eca61a1b4dfab137abcf9cdfa882 is 50, key is test_row_0/B:col10/1731408987113/Put/seqid=0 2024-11-12T10:56:29,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742411_1587 (size=12301) 2024-11-12T10:56:29,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409049647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:29,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409049647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:29,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37564 deadline: 1731409049956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:29,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T10:56:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43143 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37544 deadline: 1731409049956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 2024-11-12T10:56:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 , the old value is region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T10:56:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., hostname=aed202db7571,43143,1731408812834, seqNum=5 because the exception is null or not the one we care about 2024-11-12T10:56:30,013 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=668 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/3ae1eca61a1b4dfab137abcf9cdfa882 2024-11-12T10:56:30,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/17d969b20c474bef8da46ea84a55a381 is 50, key is test_row_0/C:col10/1731408987113/Put/seqid=0 2024-11-12T10:56:30,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742412_1588 (size=12301) 2024-11-12T10:56:30,280 INFO [Thread-2334 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2334. 2024-11-12T10:56:30,280 DEBUG [Thread-2334 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:30,280 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:30,280 INFO [Thread-2328 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2328. 2024-11-12T10:56:30,280 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:30,280 DEBUG [Thread-2328 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:30,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:30,280 DEBUG [Thread-2328 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:30,280 DEBUG [Thread-2328 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:30,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:30,281 INFO [Thread-2332 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2332. 2024-11-12T10:56:30,281 DEBUG [Thread-2332 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:30,281 DEBUG [Thread-2332 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:30,282 DEBUG [Thread-2332 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:30,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:30,283 INFO [Thread-2336 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2336. 2024-11-12T10:56:30,283 DEBUG [Thread-2336 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:30,283 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:30,283 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:30,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:30,283 INFO [Thread-2330 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2330. 2024-11-12T10:56:30,284 DEBUG [Thread-2330 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:30,284 DEBUG [Thread-2330 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:30,284 DEBUG [Thread-2330 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:30,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:30,423 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=668 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/17d969b20c474bef8da46ea84a55a381 2024-11-12T10:56:30,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/b0405b74c69b4708ac3a6472e3df8bd3 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b0405b74c69b4708ac3a6472e3df8bd3 2024-11-12T10:56:30,436 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b0405b74c69b4708ac3a6472e3df8bd3, entries=150, sequenceid=668, filesize=30.5 K 2024-11-12T10:56:30,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/3ae1eca61a1b4dfab137abcf9cdfa882 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/3ae1eca61a1b4dfab137abcf9cdfa882 2024-11-12T10:56:30,440 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/3ae1eca61a1b4dfab137abcf9cdfa882, entries=150, sequenceid=668, filesize=12.0 K 2024-11-12T10:56:30,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/17d969b20c474bef8da46ea84a55a381 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/17d969b20c474bef8da46ea84a55a381 2024-11-12T10:56:30,445 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/17d969b20c474bef8da46ea84a55a381, entries=150, sequenceid=668, filesize=12.0 K 2024-11-12T10:56:30,446 INFO [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c83771492407a48a02121732c6276538 in 1689ms, sequenceid=668, compaction requested=false 2024-11-12T10:56:30,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:30,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:30,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aed202db7571:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-12T10:56:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-12T10:56:30,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-12T10:56:30,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3680 sec 2024-11-12T10:56:30,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 3.3720 sec 2024-11-12T10:56:30,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43143 {}] regionserver.HRegion(8855): Flush requested on c83771492407a48a02121732c6276538 2024-11-12T10:56:30,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T10:56:30,471 INFO [Thread-2323 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2323. 2024-11-12T10:56:30,471 DEBUG [Thread-2323 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:30,471 INFO [Thread-2319 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2319. 2024-11-12T10:56:30,471 DEBUG [Thread-2323 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:30,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:30,471 DEBUG [Thread-2319 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:30,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:30,471 DEBUG [Thread-2319 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:30,471 DEBUG [Thread-2323 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:30,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:30,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:30,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:30,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:30,471 DEBUG [Thread-2319 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:30,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:30,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:30,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111229845a2363d14da1a10b6eedcc9f4c5f_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408989318/Put/seqid=0 2024-11-12T10:56:30,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742413_1589 (size=12454) 2024-11-12T10:56:30,883 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:30,889 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111229845a2363d14da1a10b6eedcc9f4c5f_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111229845a2363d14da1a10b6eedcc9f4c5f_c83771492407a48a02121732c6276538 2024-11-12T10:56:30,890 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/954e925be2e44f9aada5df846169d3f9, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:30,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/954e925be2e44f9aada5df846169d3f9 is 175, key is test_row_0/A:col10/1731408989318/Put/seqid=0 2024-11-12T10:56:30,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742414_1590 (size=31255) 2024-11-12T10:56:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T10:56:31,249 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:56:31,297 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=691, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/954e925be2e44f9aada5df846169d3f9 2024-11-12T10:56:31,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/d6eecc0e7480417caa8927bbcab819be is 50, key is test_row_0/B:col10/1731408989318/Put/seqid=0 2024-11-12T10:56:31,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742415_1591 (size=12301) 2024-11-12T10:56:31,378 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T10:56:31,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=691 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/d6eecc0e7480417caa8927bbcab819be 2024-11-12T10:56:31,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/2bc9792a615348bdb908ceec2e982c87 is 50, key is test_row_0/C:col10/1731408989318/Put/seqid=0 2024-11-12T10:56:31,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742416_1592 (size=12301) 2024-11-12T10:56:31,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=691 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/2bc9792a615348bdb908ceec2e982c87 2024-11-12T10:56:31,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/954e925be2e44f9aada5df846169d3f9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/954e925be2e44f9aada5df846169d3f9 2024-11-12T10:56:31,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/954e925be2e44f9aada5df846169d3f9, entries=150, sequenceid=691, filesize=30.5 K 2024-11-12T10:56:31,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/d6eecc0e7480417caa8927bbcab819be as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/d6eecc0e7480417caa8927bbcab819be 2024-11-12T10:56:31,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/d6eecc0e7480417caa8927bbcab819be, entries=150, sequenceid=691, filesize=12.0 K 2024-11-12T10:56:31,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/2bc9792a615348bdb908ceec2e982c87 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/2bc9792a615348bdb908ceec2e982c87 2024-11-12T10:56:31,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/2bc9792a615348bdb908ceec2e982c87, entries=150, sequenceid=691, filesize=12.0 K 2024-11-12T10:56:31,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for c83771492407a48a02121732c6276538 in 1275ms, sequenceid=691, compaction requested=true 2024-11-12T10:56:31,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:31,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T10:56:31,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:31,745 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:31,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T10:56:31,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:31,745 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:31,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c83771492407a48a02121732c6276538:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T10:56:31,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:31,746 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:31,746 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:31,746 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/B is initiating minor compaction (all files) 2024-11-12T10:56:31,746 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/A is initiating minor compaction (all files) 2024-11-12T10:56:31,746 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/B in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:31,746 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/A in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:31,746 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/d801ef95452c4e399b4d0f060d8ab09c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/3ae1eca61a1b4dfab137abcf9cdfa882, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/d6eecc0e7480417caa8927bbcab819be] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=36.6 K 2024-11-12T10:56:31,746 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/f7561a68f168439b98543af4c3e93edb, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b0405b74c69b4708ac3a6472e3df8bd3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/954e925be2e44f9aada5df846169d3f9] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=92.1 K 2024-11-12T10:56:31,746 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:31,746 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. files: [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/f7561a68f168439b98543af4c3e93edb, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b0405b74c69b4708ac3a6472e3df8bd3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/954e925be2e44f9aada5df846169d3f9] 2024-11-12T10:56:31,747 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting d801ef95452c4e399b4d0f060d8ab09c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1731408986443 2024-11-12T10:56:31,747 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7561a68f168439b98543af4c3e93edb, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1731408986443 2024-11-12T10:56:31,747 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ae1eca61a1b4dfab137abcf9cdfa882, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=668, earliestPutTs=1731408987107 2024-11-12T10:56:31,747 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0405b74c69b4708ac3a6472e3df8bd3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=668, earliestPutTs=1731408987107 2024-11-12T10:56:31,747 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting d6eecc0e7480417caa8927bbcab819be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=691, earliestPutTs=1731408989318 2024-11-12T10:56:31,747 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] compactions.Compactor(225): Compacting 954e925be2e44f9aada5df846169d3f9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=691, earliestPutTs=1731408989318 2024-11-12T10:56:31,755 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#B#compaction#503 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:31,755 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/de28399b28e641cba5b7739e9a9dc8f5 is 50, key is test_row_0/B:col10/1731408989318/Put/seqid=0 2024-11-12T10:56:31,757 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:31,762 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411122570a1d8569e44dc87b619b695cff669_c83771492407a48a02121732c6276538 store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:31,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742417_1593 (size=12949) 2024-11-12T10:56:31,773 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411122570a1d8569e44dc87b619b695cff669_c83771492407a48a02121732c6276538, store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:31,773 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122570a1d8569e44dc87b619b695cff669_c83771492407a48a02121732c6276538 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:31,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742418_1594 (size=4469) 2024-11-12T10:56:32,175 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/de28399b28e641cba5b7739e9a9dc8f5 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/de28399b28e641cba5b7739e9a9dc8f5 2024-11-12T10:56:32,179 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#A#compaction#504 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:32,180 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/8df0910edb5347ab85288c061994a66b is 175, key is test_row_0/A:col10/1731408989318/Put/seqid=0 2024-11-12T10:56:32,180 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/B of c83771492407a48a02121732c6276538 into de28399b28e641cba5b7739e9a9dc8f5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:32,180 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:32,180 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/B, priority=13, startTime=1731408991745; duration=0sec 2024-11-12T10:56:32,180 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T10:56:32,180 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:B 2024-11-12T10:56:32,180 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T10:56:32,181 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T10:56:32,181 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1541): c83771492407a48a02121732c6276538/C is initiating minor compaction (all files) 2024-11-12T10:56:32,181 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c83771492407a48a02121732c6276538/C in TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:32,181 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/deddd2cc35aa4a67a35a340a94c57fd6, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/17d969b20c474bef8da46ea84a55a381, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/2bc9792a615348bdb908ceec2e982c87] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp, totalSize=36.6 K 2024-11-12T10:56:32,182 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting deddd2cc35aa4a67a35a340a94c57fd6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1731408986443 2024-11-12T10:56:32,182 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 17d969b20c474bef8da46ea84a55a381, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=668, earliestPutTs=1731408987107 2024-11-12T10:56:32,182 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bc9792a615348bdb908ceec2e982c87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=691, earliestPutTs=1731408989318 2024-11-12T10:56:32,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742419_1595 (size=31903) 2024-11-12T10:56:32,192 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c83771492407a48a02121732c6276538#C#compaction#505 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:56:32,193 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/f142a669340c478c818ac2c7aaed2ea3 is 50, key is test_row_0/C:col10/1731408989318/Put/seqid=0 2024-11-12T10:56:32,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742420_1596 (size=12949) 2024-11-12T10:56:32,594 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/8df0910edb5347ab85288c061994a66b as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/8df0910edb5347ab85288c061994a66b 2024-11-12T10:56:32,600 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/A of c83771492407a48a02121732c6276538 into 8df0910edb5347ab85288c061994a66b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:32,600 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:32,600 INFO [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/A, priority=13, startTime=1731408991745; duration=0sec 2024-11-12T10:56:32,600 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:32,600 DEBUG [RS:0;aed202db7571:43143-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:A 2024-11-12T10:56:32,601 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/f142a669340c478c818ac2c7aaed2ea3 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f142a669340c478c818ac2c7aaed2ea3 2024-11-12T10:56:32,605 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c83771492407a48a02121732c6276538/C of c83771492407a48a02121732c6276538 into f142a669340c478c818ac2c7aaed2ea3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:56:32,605 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c83771492407a48a02121732c6276538: 2024-11-12T10:56:32,605 INFO [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., storeName=c83771492407a48a02121732c6276538/C, priority=13, startTime=1731408991745; duration=0sec 2024-11-12T10:56:32,605 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T10:56:32,605 DEBUG [RS:0;aed202db7571:43143-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c83771492407a48a02121732c6276538:C 2024-11-12T10:56:35,909 INFO [Thread-2321 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2321. 2024-11-12T10:56:35,909 DEBUG [Thread-2321 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:35,909 DEBUG [Thread-2321 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:35,909 DEBUG [Thread-2321 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:35,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:38,663 INFO [Thread-2317 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2317. 2024-11-12T10:56:38,664 DEBUG [Thread-2317 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:38,664 DEBUG [Thread-2317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:38,664 DEBUG [Thread-2317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:38,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:38,741 INFO [Thread-2325 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2325. 2024-11-12T10:56:38,741 DEBUG [Thread-2325 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T10:56:38,741 DEBUG [Thread-2325 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:38,741 DEBUG [Thread-2325 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:38,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:38,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T10:56:38,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-12T10:56:38,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 226 2024-11-12T10:56:38,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 161 2024-11-12T10:56:38,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 221 2024-11-12T10:56:38,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-12T10:56:38,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T10:56:38,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3442 2024-11-12T10:56:38,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3362 2024-11-12T10:56:38,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3470 2024-11-12T10:56:38,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3437 2024-11-12T10:56:38,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3343 2024-11-12T10:56:38,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T10:56:38,743 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T10:56:38,744 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T10:56:38,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:38,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:38,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:38,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T10:56:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T10:56:38,750 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731408998750"}]},"ts":"1731408998750"} 2024-11-12T10:56:38,752 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T10:56:38,752 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T10:56:38,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T10:56:38,754 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, UNASSIGN}] 2024-11-12T10:56:38,755 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, UNASSIGN 2024-11-12T10:56:38,755 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=c83771492407a48a02121732c6276538, regionState=CLOSING, regionLocation=aed202db7571,43143,1731408812834 2024-11-12T10:56:38,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, UNASSIGN because future has completed 2024-11-12T10:56:38,757 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T10:56:38,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834}] 2024-11-12T10:56:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T10:56:38,910 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close c83771492407a48a02121732c6276538 2024-11-12T10:56:38,911 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T10:56:38,911 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing c83771492407a48a02121732c6276538, disabling compactions & flushes 2024-11-12T10:56:38,911 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:38,911 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:38,911 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. after waiting 0 ms 2024-11-12T10:56:38,911 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:38,911 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing c83771492407a48a02121732c6276538 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-12T10:56:38,912 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=A 2024-11-12T10:56:38,912 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:38,912 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=B 2024-11-12T10:56:38,912 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:38,913 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c83771492407a48a02121732c6276538, store=C 2024-11-12T10:56:38,913 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T10:56:38,921 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112339170d5bd654a1da750e4400bab8e44_c83771492407a48a02121732c6276538 is 50, key is test_row_0/A:col10/1731408998659/Put/seqid=0 2024-11-12T10:56:38,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742421_1597 (size=9914) 2024-11-12T10:56:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T10:56:39,327 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:56:39,337 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112339170d5bd654a1da750e4400bab8e44_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112339170d5bd654a1da750e4400bab8e44_c83771492407a48a02121732c6276538 2024-11-12T10:56:39,338 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/f146d5c7076449889bb03d5615662c86, store: [table=TestAcidGuarantees family=A region=c83771492407a48a02121732c6276538] 2024-11-12T10:56:39,339 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/f146d5c7076449889bb03d5615662c86 is 175, key is test_row_0/A:col10/1731408998659/Put/seqid=0 2024-11-12T10:56:39,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742422_1598 (size=22561) 2024-11-12T10:56:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T10:56:39,745 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=700, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/f146d5c7076449889bb03d5615662c86 2024-11-12T10:56:39,759 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/6514caf92dd64f7989c0488666df5aaa is 50, key is test_row_0/B:col10/1731408998659/Put/seqid=0 2024-11-12T10:56:39,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742423_1599 (size=9857) 2024-11-12T10:56:39,765 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=700 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/6514caf92dd64f7989c0488666df5aaa 2024-11-12T10:56:39,773 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/f5f37e6180374b3f881462489ff21d5e is 50, key is test_row_0/C:col10/1731408998659/Put/seqid=0 2024-11-12T10:56:39,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742424_1600 (size=9857) 2024-11-12T10:56:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T10:56:40,178 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=700 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/f5f37e6180374b3f881462489ff21d5e 2024-11-12T10:56:40,189 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/A/f146d5c7076449889bb03d5615662c86 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/f146d5c7076449889bb03d5615662c86 2024-11-12T10:56:40,194 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/f146d5c7076449889bb03d5615662c86, entries=100, sequenceid=700, filesize=22.0 K 2024-11-12T10:56:40,195 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/B/6514caf92dd64f7989c0488666df5aaa as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/6514caf92dd64f7989c0488666df5aaa 2024-11-12T10:56:40,198 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/6514caf92dd64f7989c0488666df5aaa, entries=100, sequenceid=700, filesize=9.6 K 2024-11-12T10:56:40,199 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/.tmp/C/f5f37e6180374b3f881462489ff21d5e as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f5f37e6180374b3f881462489ff21d5e 2024-11-12T10:56:40,203 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f5f37e6180374b3f881462489ff21d5e, entries=100, sequenceid=700, filesize=9.6 K 2024-11-12T10:56:40,204 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c83771492407a48a02121732c6276538 in 1293ms, sequenceid=700, compaction requested=false 2024-11-12T10:56:40,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/9dcf58b0f5294d668e067d3c609c263a, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/6c181619613542bca0dd45a66699bf52, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b90573e9cb534bb3967ddc16eacd78a8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/564ecda3829c4356aea5f5caa17bc9fb, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/830f3cceb14a485db9c26ad3f993e649, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/46861e40332642e396a3a8293241f4f3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/e866473a8d35466dae898c18557c2b16, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b6cfbffb1372481ca99eb4902e44bde2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/539765ba6cc3427499abd8f7e4811675, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/042c0adfabfe4e1c9bf89c6608d62303, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/70db3aeff4d040419bb2dc41c56bc172, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/706a751267e844859aa1f6f470091566, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/05e491cf04d149e398db82e99a4ef259, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/24b6d99d01b44b54b25b7b6419ee0efc, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/1fb39cbaca6a402bac359d9dea4f5930, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/c68a1191ffe648f39a9e5c5bc98773dd, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/f7561a68f168439b98543af4c3e93edb, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b0405b74c69b4708ac3a6472e3df8bd3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/954e925be2e44f9aada5df846169d3f9] to archive 2024-11-12T10:56:40,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:56:40,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/9dcf58b0f5294d668e067d3c609c263a to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/9dcf58b0f5294d668e067d3c609c263a 2024-11-12T10:56:40,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/6c181619613542bca0dd45a66699bf52 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/6c181619613542bca0dd45a66699bf52 2024-11-12T10:56:40,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b90573e9cb534bb3967ddc16eacd78a8 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b90573e9cb534bb3967ddc16eacd78a8 2024-11-12T10:56:40,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/564ecda3829c4356aea5f5caa17bc9fb to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/564ecda3829c4356aea5f5caa17bc9fb 2024-11-12T10:56:40,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/830f3cceb14a485db9c26ad3f993e649 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/830f3cceb14a485db9c26ad3f993e649 2024-11-12T10:56:40,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/46861e40332642e396a3a8293241f4f3 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/46861e40332642e396a3a8293241f4f3 2024-11-12T10:56:40,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/e866473a8d35466dae898c18557c2b16 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/e866473a8d35466dae898c18557c2b16 2024-11-12T10:56:40,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b6cfbffb1372481ca99eb4902e44bde2 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b6cfbffb1372481ca99eb4902e44bde2 2024-11-12T10:56:40,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/539765ba6cc3427499abd8f7e4811675 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/539765ba6cc3427499abd8f7e4811675 2024-11-12T10:56:40,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/042c0adfabfe4e1c9bf89c6608d62303 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/042c0adfabfe4e1c9bf89c6608d62303 2024-11-12T10:56:40,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/70db3aeff4d040419bb2dc41c56bc172 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/70db3aeff4d040419bb2dc41c56bc172 2024-11-12T10:56:40,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/706a751267e844859aa1f6f470091566 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/706a751267e844859aa1f6f470091566 2024-11-12T10:56:40,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/05e491cf04d149e398db82e99a4ef259 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/05e491cf04d149e398db82e99a4ef259 2024-11-12T10:56:40,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/24b6d99d01b44b54b25b7b6419ee0efc to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/24b6d99d01b44b54b25b7b6419ee0efc 2024-11-12T10:56:40,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/1fb39cbaca6a402bac359d9dea4f5930 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/1fb39cbaca6a402bac359d9dea4f5930 2024-11-12T10:56:40,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/c68a1191ffe648f39a9e5c5bc98773dd to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/c68a1191ffe648f39a9e5c5bc98773dd 2024-11-12T10:56:40,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/f7561a68f168439b98543af4c3e93edb to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/f7561a68f168439b98543af4c3e93edb 2024-11-12T10:56:40,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b0405b74c69b4708ac3a6472e3df8bd3 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/b0405b74c69b4708ac3a6472e3df8bd3 2024-11-12T10:56:40,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/954e925be2e44f9aada5df846169d3f9 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/954e925be2e44f9aada5df846169d3f9 2024-11-12T10:56:40,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/f668d534bcf94feaa3cb6d45d184288c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/513e5800f3bb49529ce1e0179f5f1b16, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/94534cf3993340f6bf2624d4412fc8a2, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/29871735f5564523bf74058b7350cb2b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/7b5d226d50e54cd9af1d7b5c2ea65c13, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/c3b2aaaed1734022826db97c1ee5d961, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/f818c9de68c246a194e39d5d2df8744b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/a573678afa52478cba0d4615a13ce72b, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/bc24a8cc674244938f1a7582bc2a64ca, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/dd9b3451c51e478d91d5231ce29fc7e1, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/adc296be250b4eb7bc4cd5d6c3734ed6, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/0e6bb15814ab4d7ea4314b42511a85b5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/632bfd0dc5b64696841831027cab192c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/2885915e3a4a49ff876f67582118070f, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/91c9eed56dd642aabaa168583af0e365, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/d801ef95452c4e399b4d0f060d8ab09c, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/97b02bacb27d4fff96b400668493e31e, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/3ae1eca61a1b4dfab137abcf9cdfa882, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/d6eecc0e7480417caa8927bbcab819be] to archive 2024-11-12T10:56:40,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:56:40,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/f668d534bcf94feaa3cb6d45d184288c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/f668d534bcf94feaa3cb6d45d184288c 2024-11-12T10:56:40,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/513e5800f3bb49529ce1e0179f5f1b16 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/513e5800f3bb49529ce1e0179f5f1b16 2024-11-12T10:56:40,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/94534cf3993340f6bf2624d4412fc8a2 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/94534cf3993340f6bf2624d4412fc8a2 2024-11-12T10:56:40,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/29871735f5564523bf74058b7350cb2b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/29871735f5564523bf74058b7350cb2b 2024-11-12T10:56:40,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/7b5d226d50e54cd9af1d7b5c2ea65c13 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/7b5d226d50e54cd9af1d7b5c2ea65c13 2024-11-12T10:56:40,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/c3b2aaaed1734022826db97c1ee5d961 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/c3b2aaaed1734022826db97c1ee5d961 2024-11-12T10:56:40,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/f818c9de68c246a194e39d5d2df8744b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/f818c9de68c246a194e39d5d2df8744b 2024-11-12T10:56:40,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/a573678afa52478cba0d4615a13ce72b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/a573678afa52478cba0d4615a13ce72b 2024-11-12T10:56:40,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/bc24a8cc674244938f1a7582bc2a64ca to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/bc24a8cc674244938f1a7582bc2a64ca 2024-11-12T10:56:40,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/dd9b3451c51e478d91d5231ce29fc7e1 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/dd9b3451c51e478d91d5231ce29fc7e1 2024-11-12T10:56:40,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/adc296be250b4eb7bc4cd5d6c3734ed6 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/adc296be250b4eb7bc4cd5d6c3734ed6 2024-11-12T10:56:40,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/0e6bb15814ab4d7ea4314b42511a85b5 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/0e6bb15814ab4d7ea4314b42511a85b5 2024-11-12T10:56:40,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/632bfd0dc5b64696841831027cab192c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/632bfd0dc5b64696841831027cab192c 2024-11-12T10:56:40,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/2885915e3a4a49ff876f67582118070f to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/2885915e3a4a49ff876f67582118070f 2024-11-12T10:56:40,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/91c9eed56dd642aabaa168583af0e365 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/91c9eed56dd642aabaa168583af0e365 2024-11-12T10:56:40,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/d801ef95452c4e399b4d0f060d8ab09c to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/d801ef95452c4e399b4d0f060d8ab09c 2024-11-12T10:56:40,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/97b02bacb27d4fff96b400668493e31e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/97b02bacb27d4fff96b400668493e31e 2024-11-12T10:56:40,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/3ae1eca61a1b4dfab137abcf9cdfa882 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/3ae1eca61a1b4dfab137abcf9cdfa882 2024-11-12T10:56:40,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/d6eecc0e7480417caa8927bbcab819be to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/d6eecc0e7480417caa8927bbcab819be 2024-11-12T10:56:40,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/83a0c74214e148e9834a037a5860b843, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/b75e4ab6bb3e46afa3d726a83aea9853, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/08523b6a4b47438f903acacae65ea8d7, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/4271a256876f448fbee740ce5c239be5, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/3ec5d60cf58246ca8cddaaabe2cf6b43, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/92669762035b45b59b7b89dbd751b171, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f62ee5777c7647bca4d2312cbd8fe1bd, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/7a100265893f4dafbc7a492ad61f46d0, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/452182ddecd744b6ba211756ac3a61d8, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/fe15e0a4ce09491c817ff9089707c118, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/12d40dbd59fa456b9321a69365e79838, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f7eb42babe9a4214a390297e6cab4298, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/c6fdbae10de84477b37e19882d9eb9ab, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/7cec1232476f458690f977f83018d0b3, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/c8df7701efe14f839084bfd47349345d, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/deddd2cc35aa4a67a35a340a94c57fd6, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/2899be625eab49138fe5b59d3a27b1e9, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/17d969b20c474bef8da46ea84a55a381, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/2bc9792a615348bdb908ceec2e982c87] to archive 2024-11-12T10:56:40,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:56:40,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/83a0c74214e148e9834a037a5860b843 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/83a0c74214e148e9834a037a5860b843 2024-11-12T10:56:40,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/b75e4ab6bb3e46afa3d726a83aea9853 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/b75e4ab6bb3e46afa3d726a83aea9853 2024-11-12T10:56:40,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/08523b6a4b47438f903acacae65ea8d7 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/08523b6a4b47438f903acacae65ea8d7 2024-11-12T10:56:40,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/4271a256876f448fbee740ce5c239be5 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/4271a256876f448fbee740ce5c239be5 2024-11-12T10:56:40,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/3ec5d60cf58246ca8cddaaabe2cf6b43 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/3ec5d60cf58246ca8cddaaabe2cf6b43 2024-11-12T10:56:40,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/92669762035b45b59b7b89dbd751b171 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/92669762035b45b59b7b89dbd751b171 2024-11-12T10:56:40,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f62ee5777c7647bca4d2312cbd8fe1bd to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f62ee5777c7647bca4d2312cbd8fe1bd 2024-11-12T10:56:40,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/7a100265893f4dafbc7a492ad61f46d0 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/7a100265893f4dafbc7a492ad61f46d0 2024-11-12T10:56:40,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/452182ddecd744b6ba211756ac3a61d8 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/452182ddecd744b6ba211756ac3a61d8 2024-11-12T10:56:40,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/fe15e0a4ce09491c817ff9089707c118 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/fe15e0a4ce09491c817ff9089707c118 2024-11-12T10:56:40,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/12d40dbd59fa456b9321a69365e79838 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/12d40dbd59fa456b9321a69365e79838 2024-11-12T10:56:40,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f7eb42babe9a4214a390297e6cab4298 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f7eb42babe9a4214a390297e6cab4298 2024-11-12T10:56:40,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/c6fdbae10de84477b37e19882d9eb9ab to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/c6fdbae10de84477b37e19882d9eb9ab 2024-11-12T10:56:40,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/7cec1232476f458690f977f83018d0b3 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/7cec1232476f458690f977f83018d0b3 2024-11-12T10:56:40,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/c8df7701efe14f839084bfd47349345d to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/c8df7701efe14f839084bfd47349345d 2024-11-12T10:56:40,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/deddd2cc35aa4a67a35a340a94c57fd6 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/deddd2cc35aa4a67a35a340a94c57fd6 2024-11-12T10:56:40,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/2899be625eab49138fe5b59d3a27b1e9 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/2899be625eab49138fe5b59d3a27b1e9 2024-11-12T10:56:40,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/17d969b20c474bef8da46ea84a55a381 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/17d969b20c474bef8da46ea84a55a381 2024-11-12T10:56:40,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/2bc9792a615348bdb908ceec2e982c87 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/2bc9792a615348bdb908ceec2e982c87 2024-11-12T10:56:40,266 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/recovered.edits/703.seqid, newMaxSeqId=703, maxSeqId=4 2024-11-12T10:56:40,266 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. 2024-11-12T10:56:40,267 DEBUG [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for c83771492407a48a02121732c6276538: Waiting for close lock at 1731408998911Running coprocessor pre-close hooks at 1731408998911Disabling compacts and flushes for region at 1731408998911Disabling writes for close at 1731408998911Obtaining lock to block concurrent updates at 1731408998912 (+1 ms)Preparing flush snapshotting stores in c83771492407a48a02121732c6276538 at 1731408998912Finished memstore snapshotting TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731408998913 (+1 ms)Flushing stores of TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538. at 1731408998914 (+1 ms)Flushing c83771492407a48a02121732c6276538/A: creating writer at 1731408998914Flushing mob file c83771492407a48a02121732c6276538/A: appending metadata at 1731408998921 (+7 ms)Flushing mob file c83771492407a48a02121732c6276538/A: closing flushed file at 1731408998921Flushing c83771492407a48a02121732c6276538/A: appending metadata at 1731408999338 (+417 ms)Flushing c83771492407a48a02121732c6276538/A: closing flushed file at 1731408999338Flushing c83771492407a48a02121732c6276538/B: creating writer at 1731408999755 (+417 ms)Flushing c83771492407a48a02121732c6276538/B: appending metadata at 1731408999759 (+4 ms)Flushing c83771492407a48a02121732c6276538/B: closing flushed file at 1731408999759Flushing c83771492407a48a02121732c6276538/C: creating writer at 1731408999769 (+10 ms)Flushing c83771492407a48a02121732c6276538/C: appending metadata at 1731408999773 (+4 ms)Flushing c83771492407a48a02121732c6276538/C: closing flushed file at 1731408999773Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d1982f1: reopening flushed file at 1731409000187 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bbd8ea6: reopening flushed file at 1731409000194 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b5f2804: reopening flushed file at 1731409000198 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c83771492407a48a02121732c6276538 in 1293ms, sequenceid=700, compaction requested=false at 1731409000204 (+6 ms)Writing region close event to WAL at 1731409000264 (+60 ms)Running coprocessor post-close hooks at 1731409000266 (+2 ms)Closed at 1731409000266 2024-11-12T10:56:40,268 INFO [RS_CLOSE_REGION-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed c83771492407a48a02121732c6276538 2024-11-12T10:56:40,269 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=c83771492407a48a02121732c6276538, regionState=CLOSED 2024-11-12T10:56:40,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 because future has completed 2024-11-12T10:56:40,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-12T10:56:40,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure c83771492407a48a02121732c6276538, server=aed202db7571,43143,1731408812834 in 1.5130 sec 2024-11-12T10:56:40,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-12T10:56:40,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c83771492407a48a02121732c6276538, UNASSIGN in 1.5170 sec 2024-11-12T10:56:40,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-12T10:56:40,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5200 sec 2024-11-12T10:56:40,274 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731409000274"}]},"ts":"1731409000274"} 2024-11-12T10:56:40,275 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T10:56:40,275 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T10:56:40,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5300 sec 2024-11-12T10:56:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T10:56:40,907 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:56:40,908 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T10:56:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:40,909 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-12T10:56:40,910 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:40,912 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538 2024-11-12T10:56:40,913 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/recovered.edits] 2024-11-12T10:56:40,915 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/8df0910edb5347ab85288c061994a66b to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/8df0910edb5347ab85288c061994a66b 2024-11-12T10:56:40,916 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/f146d5c7076449889bb03d5615662c86 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/A/f146d5c7076449889bb03d5615662c86 2024-11-12T10:56:40,919 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/6514caf92dd64f7989c0488666df5aaa to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/6514caf92dd64f7989c0488666df5aaa 2024-11-12T10:56:40,920 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/de28399b28e641cba5b7739e9a9dc8f5 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/B/de28399b28e641cba5b7739e9a9dc8f5 2024-11-12T10:56:40,922 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f142a669340c478c818ac2c7aaed2ea3 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f142a669340c478c818ac2c7aaed2ea3 2024-11-12T10:56:40,923 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f5f37e6180374b3f881462489ff21d5e to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/C/f5f37e6180374b3f881462489ff21d5e 2024-11-12T10:56:40,927 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/recovered.edits/703.seqid to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538/recovered.edits/703.seqid 2024-11-12T10:56:40,927 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/default/TestAcidGuarantees/c83771492407a48a02121732c6276538 2024-11-12T10:56:40,927 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T10:56:40,928 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T10:56:40,928 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T10:56:40,932 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111202548d0e847d44b7a0a6c999d0360ed3_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111202548d0e847d44b7a0a6c999d0360ed3_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,933 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111229845a2363d14da1a10b6eedcc9f4c5f_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111229845a2363d14da1a10b6eedcc9f4c5f_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,934 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111229f858d95a1c45f7a75447eed3dd3219_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111229f858d95a1c45f7a75447eed3dd3219_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,936 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112339170d5bd654a1da750e4400bab8e44_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112339170d5bd654a1da750e4400bab8e44_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,937 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126577305f5ef7449ebf94e9176cca7c95_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126577305f5ef7449ebf94e9176cca7c95_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,939 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126e230ac3c49e46719fd6200e8b6cde90_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126e230ac3c49e46719fd6200e8b6cde90_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,941 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112850eb8e4127b4b13a087920026628363_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112850eb8e4127b4b13a087920026628363_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,943 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112874a9354f4d547eeaa3048e9c0673c86_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112874a9354f4d547eeaa3048e9c0673c86_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,944 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111295a956a59cc74f2e9d799ee5e0cfb380_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111295a956a59cc74f2e9d799ee5e0cfb380_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,946 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112dbdd30ce874949119ad5ee188aef8730_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112dbdd30ce874949119ad5ee188aef8730_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,948 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e21f25f40ffc47abb1a6ff767f2a45b2_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e21f25f40ffc47abb1a6ff767f2a45b2_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,949 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e69642c76c1143bea55c4c2ddfd4221d_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e69642c76c1143bea55c4c2ddfd4221d_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,950 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ec1594b6e8dc4a16b469688f3ed29b87_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ec1594b6e8dc4a16b469688f3ed29b87_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,951 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f2ca60a81fd6434c9a5b3f33fc5292ac_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f2ca60a81fd6434c9a5b3f33fc5292ac_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,952 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f4f4dfc1a75e41488ffcfbb6e2845149_c83771492407a48a02121732c6276538 to hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f4f4dfc1a75e41488ffcfbb6e2845149_c83771492407a48a02121732c6276538 2024-11-12T10:56:40,952 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T10:56:40,954 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:40,956 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T10:56:40,957 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T10:56:40,958 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:40,958 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T10:56:40,959 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731409000958"}]},"ts":"9223372036854775807"} 2024-11-12T10:56:40,960 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T10:56:40,960 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c83771492407a48a02121732c6276538, NAME => 'TestAcidGuarantees,,1731408967131.c83771492407a48a02121732c6276538.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T10:56:40,960 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T10:56:40,960 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731409000960"}]},"ts":"9223372036854775807"} 2024-11-12T10:56:40,961 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T10:56:40,962 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T10:56:40,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 54 msec 2024-11-12T10:56:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-12T10:56:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T10:56:41,019 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T10:56:41,034 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 252), OpenFileDescriptor=449 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=359 (was 427), ProcessCount=11 (was 11), AvailableMemoryMB=2217 (was 2318) 2024-11-12T10:56:41,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T10:56:41,034 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T10:56:41,034 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T10:56:41,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:41,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:41,034 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:56:41,035 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T10:56:41,035 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1831818974, stopped=false 2024-11-12T10:56:41,035 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=aed202db7571,45901,1731408812240 2024-11-12T10:56:41,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T10:56:41,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T10:56:41,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:56:41,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:56:41,037 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T10:56:41,037 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T10:56:41,037 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T10:56:41,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:41,038 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T10:56:41,038 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T10:56:41,038 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'aed202db7571,43143,1731408812834' ***** 2024-11-12T10:56:41,038 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T10:56:41,038 INFO [RS:0;aed202db7571:43143 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T10:56:41,039 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T10:56:41,039 INFO [RS:0;aed202db7571:43143 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T10:56:41,039 INFO [RS:0;aed202db7571:43143 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T10:56:41,039 INFO [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(959): stopping server aed202db7571,43143,1731408812834 2024-11-12T10:56:41,039 INFO [RS:0;aed202db7571:43143 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T10:56:41,039 INFO [RS:0;aed202db7571:43143 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;aed202db7571:43143. 2024-11-12T10:56:41,039 DEBUG [RS:0;aed202db7571:43143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T10:56:41,039 DEBUG [RS:0;aed202db7571:43143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:41,039 INFO [RS:0;aed202db7571:43143 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T10:56:41,039 INFO [RS:0;aed202db7571:43143 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T10:56:41,039 INFO [RS:0;aed202db7571:43143 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T10:56:41,040 INFO [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T10:56:41,040 INFO [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T10:56:41,040 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-12T10:56:41,040 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T10:56:41,040 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T10:56:41,040 INFO [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T10:56:41,040 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T10:56:41,040 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T10:56:41,040 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T10:56:41,041 INFO [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-12T10:56:41,062 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/.tmp/info/8f8aa0b289a144be8bb26aeb3db040de is 95, key is TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055./info:/1731408842972/DeleteFamily/seqid=0 2024-11-12T10:56:41,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742425_1601 (size=6514) 2024-11-12T10:56:41,085 INFO [regionserver/aed202db7571:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T10:56:41,241 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T10:56:41,441 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T10:56:41,467 INFO [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/.tmp/info/8f8aa0b289a144be8bb26aeb3db040de 2024-11-12T10:56:41,494 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/.tmp/ns/87a4690a37fe436b935020fd46c57e60 is 93, key is TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055./ns:/1731408842925/DeleteFamily/seqid=0 2024-11-12T10:56:41,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742426_1602 (size=5906) 2024-11-12T10:56:41,524 INFO [regionserver/aed202db7571:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T10:56:41,524 INFO [regionserver/aed202db7571:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T10:56:41,641 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T10:56:41,841 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T10:56:41,897 INFO [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/.tmp/ns/87a4690a37fe436b935020fd46c57e60 2024-11-12T10:56:41,922 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/.tmp/rep_barrier/1eff7beb149147dab7cd0bec291c36a4 is 102, key is TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055./rep_barrier:/1731408842925/DeleteFamily/seqid=0 2024-11-12T10:56:41,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742427_1603 (size=6025) 2024-11-12T10:56:42,042 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T10:56:42,242 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T10:56:42,330 INFO [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/.tmp/rep_barrier/1eff7beb149147dab7cd0bec291c36a4 2024-11-12T10:56:42,356 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/.tmp/table/0a01d94bd0e84259939b80262779cf3d is 96, key is TestAcidGuarantees,,1731408816649.b3cb1a6227d163d89478d6e9738e8055./table:/1731408842925/DeleteFamily/seqid=0 2024-11-12T10:56:42,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742428_1604 (size=5936) 2024-11-12T10:56:42,442 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T10:56:42,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T10:56:42,643 DEBUG [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T10:56:42,761 INFO [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/.tmp/table/0a01d94bd0e84259939b80262779cf3d 2024-11-12T10:56:42,771 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/.tmp/info/8f8aa0b289a144be8bb26aeb3db040de as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/info/8f8aa0b289a144be8bb26aeb3db040de 2024-11-12T10:56:42,776 INFO [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/info/8f8aa0b289a144be8bb26aeb3db040de, entries=12, sequenceid=90, filesize=6.4 K 2024-11-12T10:56:42,777 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/.tmp/ns/87a4690a37fe436b935020fd46c57e60 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/ns/87a4690a37fe436b935020fd46c57e60 2024-11-12T10:56:42,783 INFO [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/ns/87a4690a37fe436b935020fd46c57e60, entries=8, sequenceid=90, filesize=5.8 K 2024-11-12T10:56:42,784 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/.tmp/rep_barrier/1eff7beb149147dab7cd0bec291c36a4 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/rep_barrier/1eff7beb149147dab7cd0bec291c36a4 2024-11-12T10:56:42,789 INFO [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/rep_barrier/1eff7beb149147dab7cd0bec291c36a4, entries=6, sequenceid=90, filesize=5.9 K 2024-11-12T10:56:42,790 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/.tmp/table/0a01d94bd0e84259939b80262779cf3d as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/table/0a01d94bd0e84259939b80262779cf3d 2024-11-12T10:56:42,795 INFO [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/table/0a01d94bd0e84259939b80262779cf3d, entries=7, sequenceid=90, filesize=5.8 K 2024-11-12T10:56:42,796 INFO [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1756ms, sequenceid=90, compaction requested=false 2024-11-12T10:56:42,801 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-12T10:56:42,802 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T10:56:42,802 INFO [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T10:56:42,802 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731409001040Running coprocessor pre-close hooks at 1731409001040Disabling compacts and flushes for region at 1731409001040Disabling writes for close at 1731409001040Obtaining lock to block concurrent updates at 1731409001041 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731409001041Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731409001041Flushing stores of hbase:meta,,1.1588230740 at 1731409001041Flushing 1588230740/info: creating writer at 1731409001042 (+1 ms)Flushing 1588230740/info: appending metadata at 1731409001062 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731409001062Flushing 1588230740/ns: creating writer at 1731409001478 (+416 ms)Flushing 1588230740/ns: appending metadata at 1731409001493 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731409001494 (+1 ms)Flushing 1588230740/rep_barrier: creating writer at 1731409001901 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731409001922 (+21 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731409001922Flushing 1588230740/table: creating writer at 1731409002339 (+417 ms)Flushing 1588230740/table: appending metadata at 1731409002356 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731409002356Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45c33cf: reopening flushed file at 1731409002770 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@584dd97d: reopening flushed file at 1731409002776 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b1f9591: reopening flushed file at 1731409002783 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@703859b3: reopening flushed file at 1731409002789 (+6 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1756ms, sequenceid=90, compaction requested=false at 1731409002796 (+7 ms)Writing region close event to WAL at 1731409002798 (+2 ms)Running coprocessor post-close hooks at 1731409002802 (+4 ms)Closed at 1731409002802 2024-11-12T10:56:42,802 DEBUG [RS_CLOSE_META-regionserver/aed202db7571:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T10:56:42,843 INFO [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(976): stopping server aed202db7571,43143,1731408812834; all regions closed. 2024-11-12T10:56:42,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741834_1010 (size=25344) 2024-11-12T10:56:42,856 DEBUG [RS:0;aed202db7571:43143 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/oldWALs 2024-11-12T10:56:42,856 INFO [RS:0;aed202db7571:43143 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL aed202db7571%2C43143%2C1731408812834.meta:.meta(num 1731408816262) 2024-11-12T10:56:42,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741833_1009 (size=16970021) 2024-11-12T10:56:42,861 DEBUG [RS:0;aed202db7571:43143 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/oldWALs 2024-11-12T10:56:42,861 INFO [RS:0;aed202db7571:43143 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL aed202db7571%2C43143%2C1731408812834:(num 1731408815650) 2024-11-12T10:56:42,861 DEBUG [RS:0;aed202db7571:43143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:56:42,861 INFO [RS:0;aed202db7571:43143 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T10:56:42,861 INFO [RS:0;aed202db7571:43143 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T10:56:42,862 INFO [RS:0;aed202db7571:43143 {}] hbase.ChoreService(370): Chore service for: regionserver/aed202db7571:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T10:56:42,862 INFO [RS:0;aed202db7571:43143 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T10:56:42,862 INFO [regionserver/aed202db7571:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T10:56:42,863 INFO [RS:0;aed202db7571:43143 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43143 2024-11-12T10:56:42,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T10:56:42,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/aed202db7571,43143,1731408812834 2024-11-12T10:56:42,867 INFO [RS:0;aed202db7571:43143 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T10:56:42,868 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [aed202db7571,43143,1731408812834] 2024-11-12T10:56:42,869 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/aed202db7571,43143,1731408812834 already deleted, retry=false 2024-11-12T10:56:42,869 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; aed202db7571,43143,1731408812834 expired; onlineServers=0 2024-11-12T10:56:42,869 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'aed202db7571,45901,1731408812240' ***** 2024-11-12T10:56:42,869 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T10:56:42,869 INFO [M:0;aed202db7571:45901 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T10:56:42,870 INFO [M:0;aed202db7571:45901 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T10:56:42,870 DEBUG [M:0;aed202db7571:45901 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T10:56:42,870 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T10:56:42,870 DEBUG [M:0;aed202db7571:45901 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T10:56:42,870 DEBUG [master/aed202db7571:0:becomeActiveMaster-HFileCleaner.large.0-1731408815386 {}] cleaner.HFileCleaner(306): Exit Thread[master/aed202db7571:0:becomeActiveMaster-HFileCleaner.large.0-1731408815386,5,FailOnTimeoutGroup] 2024-11-12T10:56:42,870 DEBUG [master/aed202db7571:0:becomeActiveMaster-HFileCleaner.small.0-1731408815390 {}] cleaner.HFileCleaner(306): Exit Thread[master/aed202db7571:0:becomeActiveMaster-HFileCleaner.small.0-1731408815390,5,FailOnTimeoutGroup] 2024-11-12T10:56:42,870 INFO [M:0;aed202db7571:45901 {}] hbase.ChoreService(370): Chore service for: master/aed202db7571:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T10:56:42,870 INFO [M:0;aed202db7571:45901 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T10:56:42,870 DEBUG [M:0;aed202db7571:45901 {}] master.HMaster(1795): Stopping service threads 2024-11-12T10:56:42,870 INFO [M:0;aed202db7571:45901 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T10:56:42,870 INFO [M:0;aed202db7571:45901 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T10:56:42,871 ERROR [M:0;aed202db7571:45901 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-12T10:56:42,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T10:56:42,871 INFO [M:0;aed202db7571:45901 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T10:56:42,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:56:42,871 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T10:56:42,872 DEBUG [M:0;aed202db7571:45901 {}] zookeeper.ZKUtil(347): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T10:56:42,872 WARN [M:0;aed202db7571:45901 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T10:56:42,872 INFO [M:0;aed202db7571:45901 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/.lastflushedseqids 2024-11-12T10:56:42,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742429_1605 (size=72) 2024-11-12T10:56:42,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T10:56:42,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43143-0x100339132190001, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T10:56:42,970 INFO [RS:0;aed202db7571:43143 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T10:56:42,970 INFO [RS:0;aed202db7571:43143 {}] regionserver.HRegionServer(1031): Exiting; stopping=aed202db7571,43143,1731408812834; zookeeper connection closed. 2024-11-12T10:56:42,971 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7d4b919a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7d4b919a 2024-11-12T10:56:42,972 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-12T10:56:43,284 INFO [M:0;aed202db7571:45901 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T10:56:43,285 INFO [M:0;aed202db7571:45901 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T10:56:43,285 DEBUG [M:0;aed202db7571:45901 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T10:56:43,285 INFO [M:0;aed202db7571:45901 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:56:43,285 DEBUG [M:0;aed202db7571:45901 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:56:43,285 DEBUG [M:0;aed202db7571:45901 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T10:56:43,286 DEBUG [M:0;aed202db7571:45901 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:56:43,286 INFO [M:0;aed202db7571:45901 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=804.56 KB heapSize=988.30 KB 2024-11-12T10:56:43,308 DEBUG [M:0;aed202db7571:45901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c38115f8a38840c7aa3aca4938857063 is 82, key is hbase:meta,,1/info:regioninfo/1731408816340/Put/seqid=0 2024-11-12T10:56:43,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742430_1606 (size=5672) 2024-11-12T10:56:43,712 INFO [M:0;aed202db7571:45901 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2280 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c38115f8a38840c7aa3aca4938857063 2024-11-12T10:56:43,740 DEBUG [M:0;aed202db7571:45901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ce24c9d60254775bdf2e04e654871d4 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1731408970238/Put/seqid=0 2024-11-12T10:56:43,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742431_1607 (size=45346) 2024-11-12T10:56:44,145 INFO [M:0;aed202db7571:45901 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=804 KB at sequenceid=2280 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ce24c9d60254775bdf2e04e654871d4 2024-11-12T10:56:44,154 INFO [M:0;aed202db7571:45901 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7ce24c9d60254775bdf2e04e654871d4 2024-11-12T10:56:44,172 DEBUG [M:0;aed202db7571:45901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e98464864f1643c196257ae1864a4ee9 is 69, key is aed202db7571,43143,1731408812834/rs:state/1731408815412/Put/seqid=0 2024-11-12T10:56:44,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073742432_1608 (size=5156) 2024-11-12T10:56:44,577 INFO [M:0;aed202db7571:45901 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2280 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e98464864f1643c196257ae1864a4ee9 2024-11-12T10:56:44,586 DEBUG [M:0;aed202db7571:45901 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c38115f8a38840c7aa3aca4938857063 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c38115f8a38840c7aa3aca4938857063 2024-11-12T10:56:44,591 INFO [M:0;aed202db7571:45901 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c38115f8a38840c7aa3aca4938857063, entries=8, sequenceid=2280, filesize=5.5 K 2024-11-12T10:56:44,592 DEBUG [M:0;aed202db7571:45901 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ce24c9d60254775bdf2e04e654871d4 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7ce24c9d60254775bdf2e04e654871d4 2024-11-12T10:56:44,596 INFO [M:0;aed202db7571:45901 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7ce24c9d60254775bdf2e04e654871d4 2024-11-12T10:56:44,596 INFO [M:0;aed202db7571:45901 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7ce24c9d60254775bdf2e04e654871d4, entries=170, sequenceid=2280, filesize=44.3 K 2024-11-12T10:56:44,597 DEBUG [M:0;aed202db7571:45901 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e98464864f1643c196257ae1864a4ee9 as hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e98464864f1643c196257ae1864a4ee9 2024-11-12T10:56:44,599 INFO [M:0;aed202db7571:45901 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/e58832fb-6285-3517-50c0-aa77a1b30d98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e98464864f1643c196257ae1864a4ee9, entries=1, sequenceid=2280, filesize=5.0 K 2024-11-12T10:56:44,600 INFO [M:0;aed202db7571:45901 {}] regionserver.HRegion(3140): Finished flush of dataSize ~804.56 KB/823865, heapSize ~988 KB/1011712, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1314ms, sequenceid=2280, compaction requested=false 2024-11-12T10:56:44,601 INFO [M:0;aed202db7571:45901 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:56:44,601 DEBUG [M:0;aed202db7571:45901 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731409003285Disabling compacts and flushes for region at 1731409003285Disabling writes for close at 1731409003286 (+1 ms)Obtaining lock to block concurrent updates at 1731409003286Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731409003286Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=823865, getHeapSize=1011952, getOffHeapSize=0, getCellsCount=2475 at 1731409003287 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731409003288 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731409003288Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731409003308 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731409003308Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731409003717 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731409003740 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731409003740Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731409004154 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731409004171 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731409004172 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2338e360: reopening flushed file at 1731409004585 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b4bdada: reopening flushed file at 1731409004591 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cbb3f1d: reopening flushed file at 1731409004596 (+5 ms)Finished flush of dataSize ~804.56 KB/823865, heapSize ~988 KB/1011712, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1314ms, sequenceid=2280, compaction requested=false at 1731409004600 (+4 ms)Writing region close event to WAL at 1731409004601 (+1 ms)Closed at 1731409004601 2024-11-12T10:56:44,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43441 is added to blk_1073741830_1006 (size=973264) 2024-11-12T10:56:44,604 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T10:56:44,604 INFO [M:0;aed202db7571:45901 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T10:56:44,604 INFO [M:0;aed202db7571:45901 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45901 2024-11-12T10:56:44,604 INFO [M:0;aed202db7571:45901 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T10:56:44,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T10:56:44,706 INFO [M:0;aed202db7571:45901 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T10:56:44,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45901-0x100339132190000, quorum=127.0.0.1:59192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T10:56:44,717 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T10:56:44,720 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T10:56:44,721 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T10:56:44,721 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T10:56:44,721 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/hadoop.log.dir/,STOPPED} 2024-11-12T10:56:44,724 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T10:56:44,724 WARN [BP-1014935059-172.17.0.2-1731408809727 heartbeating to localhost/127.0.0.1:37247 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T10:56:44,724 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T10:56:44,724 WARN [BP-1014935059-172.17.0.2-1731408809727 heartbeating to localhost/127.0.0.1:37247 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1014935059-172.17.0.2-1731408809727 (Datanode Uuid 901d1eb8-d5ba-4dfd-bff8-76a6f9bf2888) service to localhost/127.0.0.1:37247 2024-11-12T10:56:44,726 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/cluster_a82bd1d2-3d77-b394-fc32-98907a12be25/data/data1/current/BP-1014935059-172.17.0.2-1731408809727 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T10:56:44,727 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/cluster_a82bd1d2-3d77-b394-fc32-98907a12be25/data/data2/current/BP-1014935059-172.17.0.2-1731408809727 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T10:56:44,727 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T10:56:44,733 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T10:56:44,734 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T10:56:44,734 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T10:56:44,734 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T10:56:44,734 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8759adee-9ea3-9ec7-9e1a-9a490b13a96e/hadoop.log.dir/,STOPPED} 2024-11-12T10:56:44,747 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T10:56:44,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down